Changeset 15518 for trunk/sources/HeuristicLab.DataPreprocessing

Timestamp:

12/12/17 16:32:35 (7 years ago)

Author:

pfleck

Message:

#2809: merged branch to trunk

Location:

trunk/sources/HeuristicLab.DataPreprocessing

Files:

• . (modified) (1 prop)
• 3.4 (modified) (1 prop)
• 3.4/Content/CorrelationMatrixContent.cs (modified) (3 diffs)
• 3.4/Content/DataCompletenessChartContent.cs (modified) (1 diff)
• 3.4/Content/DataGridContent.cs (modified) (4 diffs)
• 3.4/Content/FilterContent.cs (modified) (1 diff)
• 3.4/Content/HistogramContent.cs (modified) (3 diffs)
• 3.4/Content/LineChartContent.cs (modified) (2 diffs)
• 3.4/Content/ManipulationContent.cs (modified) (1 diff)
• 3.4/Content/MultiScatterPlotContent.cs (modified) (2 diffs)
• 3.4/Content/PreprocessingChartContent.cs (modified) (4 diffs)
• 3.4/Content/PreprocessingContent.cs (copied) (copied from branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingContent.cs)
• 3.4/Content/ScatterPlotContent.cs (modified) (1 diff)
• 3.4/Content/SingleScatterPlotContent.cs (modified) (2 diffs)
• 3.4/Content/StatisticsContent.cs (modified) (1 diff)
• 3.4/Content/TransformationContent.cs (modified) (1 diff)
• 3.4/Data/FilteredPreprocessingData.cs (modified) (6 diffs)
• 3.4/Data/IFilteredPreprocessingData.cs (modified) (2 diffs)
• 3.4/Data/IPreprocessingData.cs (modified) (4 diffs)
• 3.4/Data/ITransactionalPreprocessingData.cs (deleted)
• 3.4/Data/PreprocessingData.cs (modified) (6 diffs)
• 3.4/Data/TransactionalPreprocessingData.cs (deleted)
• 3.4/Filter (copied) (copied from branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Filter)
• 3.4/HeuristicLab.DataPreprocessing-3.4.csproj (modified) (3 diffs)
• 3.4/Logic (deleted)
• 3.4/PreprocessingContext.cs (modified) (1 diff)
• 3.4/PreprocessingTransformator.cs (modified) (3 diffs)

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/CorrelationMatrixContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Feature Correlation Matrix", "Represents the feature correlation matrix.")]
-  public class CorrelationMatrixContent : Item, IViewShortcut {
+  [StorableClass]
+  public class CorrelationMatrixContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Gradient; }
     }
 
+    [Storable]
     public PreprocessingContext Context { get; private set; }
-    public ITransactionalPreprocessingData PreprocessingData {
-      get { return Context.Data; }
-    }
+
 
     public DataAnalysisProblemData ProblemData {
@@ -47 +48 @@
     }
 
-    public CorrelationMatrixContent(PreprocessingContext context) {
+    #region Constructor, Cloning & Persistence
+    public CorrelationMatrixContent(PreprocessingContext context)
+      : base(context.Data) {
       Context = context;
     }
@@ -55 +58 @@
       Context = original.Context;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new CorrelationMatrixContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected CorrelationMatrixContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public event DataPreprocessingChangedEventHandler Changed {

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/DataCompletenessChartContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Data Completeness Chart", "Represents a datacompleteness chart.")]
-
-  public class DataCompletenessChartContent : Item, IViewShortcut {
+  [StorableClass]
+  public class DataCompletenessChartContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.EditBrightnessContrast; }
     }
 
-    public SearchLogic SearchLogic { get; private set; }
-
-    public DataCompletenessChartContent(SearchLogic searchLogic) {
-      SearchLogic = searchLogic;
+    #region Constructor, Cloning & Persistence
+    public DataCompletenessChartContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
     }
 
     public DataCompletenessChartContent(DataCompletenessChartContent content, Cloner cloner)
       : base(content, cloner) {
-      SearchLogic = content.SearchLogic;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new DataCompletenessChartContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected DataCompletenessChartContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs

@@ -27 +27 @@
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Random;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Data Grid", "Represents a data grid.")]
-  public class DataGridContent : Item, IStringConvertibleMatrix, IViewShortcut {
+  [StorableClass]
+  public class DataGridContent : PreprocessingContent, IStringConvertibleMatrix, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Table; }
     }
 
-    public ITransactionalPreprocessingData PreProcessingData { get; private set; }
-
-    public ManipulationLogic ManipulationLogic { get; private set; }
-    public FilterLogic FilterLogic { get; private set; }
-
     public int Rows {
-      get { return PreProcessingData.Rows; }
+      get { return PreprocessingData.Rows; }
       set { }
     }
 
     public int Columns {
-      get { return PreProcessingData.Columns; }
+      get { return PreprocessingData.Columns; }
       set { }
     }
 
     public IEnumerable<string> ColumnNames {
-      get { return PreProcessingData.VariableNames; }
+      get { return PreprocessingData.VariableNames; }
       set { }
     }
@@ -71 +68 @@
 
     public IDictionary<int, IList<int>> Selection {
-      get { return PreProcessingData.Selection; }
-      set { PreProcessingData.Selection = value; }
-    }
-
-    public DataGridContent(ITransactionalPreprocessingData preProcessingData, ManipulationLogic theManipulationLogic, FilterLogic theFilterLogic) {
-      ManipulationLogic = theManipulationLogic;
-      FilterLogic = theFilterLogic;
-      PreProcessingData = preProcessingData;
-    }
-
-    public DataGridContent(DataGridContent dataGridContent, Cloner cloner)
-      : base(dataGridContent, cloner) {
-
+      get { return PreprocessingData.Selection; }
+      set { PreprocessingData.Selection = value; }
+    }
+
+    #region Constructor, Cloning & Persistence
+    public DataGridContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
+    }
+
+    public DataGridContent(DataGridContent original, Cloner cloner)
+      : base(original, cloner) {
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -89 +84 @@
     }
 
+    [StorableConstructor]
+    protected DataGridContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
+
     public void DeleteRows(IEnumerable<int> rows) {
-      PreProcessingData.DeleteRowsWithIndices(rows);
+      PreprocessingData.DeleteRowsWithIndices(rows);
     }
 
     public void DeleteColumn(int column) {
-      PreProcessingData.DeleteColumn(column);
+      PreprocessingData.DeleteColumn(column);
     }
 
     public bool Validate(string value, out string errorMessage, int columnIndex) {
-      return PreProcessingData.Validate(value, out errorMessage, columnIndex);
+      return PreprocessingData.Validate(value, out errorMessage, columnIndex);
     }
 
     public string GetValue(int rowIndex, int columnIndex) {
-      return PreProcessingData.GetCellAsString(columnIndex, rowIndex);
+      return PreprocessingData.GetCellAsString(columnIndex, rowIndex);
     }
 
     public bool SetValue(string value, int rowIndex, int columnIndex) {
-      return PreProcessingData.SetValue(value, columnIndex, rowIndex);
+      return PreprocessingData.SetValue(value, columnIndex, rowIndex);
     }
 
     public event DataPreprocessingChangedEventHandler Changed {
-      add { PreProcessingData.Changed += value; }
-      remove { PreProcessingData.Changed -= value; }
+      add { PreprocessingData.Changed += value; }
+      remove { PreprocessingData.Changed -= value; }
     }
 
@@ -132 +132 @@
 #pragma warning restore 0067
     #endregion
+
+    #region Manipulations
+    private void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, Func<int, double> doubleAggregator = null,
+      Func<int, DateTime> dateTimeAggregator = null, Func<int, string> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var value = doubleAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<double>(column.Key, index, value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var value = dateTimeAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<DateTime>(column.Key, index, value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var value = stringAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<string>(column.Key, index, value);
+          }
+        }
+      });
+    }
+
+    private void ReplaceIndicesByValues(IDictionary<int, IList<int>> cells, Func<int, IEnumerable<double>> doubleAggregator = null,
+      Func<int, IEnumerable<DateTime>> dateTimeAggregator = null, Func<int, IEnumerable<string>> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var values = doubleAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<double>(column.Key, pair.row, pair.value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var values = dateTimeAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<DateTime>(column.Key, pair.row, pair.value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var values = stringAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<string>(column.Key, pair.row, pair.value);
+          }
+        }
+      });
+    }
+
+    public void ReplaceIndicesByMean(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMean<double>(col, considerSelection),
+        col => PreprocessingData.GetMean<DateTime>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMedian<double>(col, considerSelection),
+        col => PreprocessingData.GetMedian<DateTime>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByMode(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMode<double>(col, considerSelection),
+        col => PreprocessingData.GetMode<DateTime>(col, considerSelection),
+        col => PreprocessingData.GetMode<string>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      var rand = new FastRandom();
+      ReplaceIndicesByValues(cells,
+        col => {
+          double min = PreprocessingData.GetMin<double>(col, considerSelection);
+          double max = PreprocessingData.GetMax<double>(col, considerSelection);
+          double range = max - min;
+          return cells[col].Select(_ => rand.NextDouble() * range + min);
+        },
+        col => {
+          var min = PreprocessingData.GetMin<DateTime>(col, considerSelection);
+          var max = PreprocessingData.GetMax<DateTime>(col, considerSelection);
+          double range = (max - min).TotalSeconds;
+          return cells[col].Select(_ => min + TimeSpan.FromSeconds(rand.NextDouble() * range));
+        });
+    }
+
+    public void ReplaceIndicesByString(IDictionary<int, IList<int>> cells, string value) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          foreach (var rowIdx in column.Value) {
+            PreprocessingData.SetValue(value, column.Key, rowIdx);
+          }
+        }
+      });
+    }
+
+
+    public void ReplaceIndicesByLinearInterpolationOfNeighbours(IDictionary<int, IList<int>> cells) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          IList<Tuple<int, int>> startEndings = GetStartAndEndingsForInterpolation(column);
+          foreach (var tuple in startEndings) {
+            Interpolate(column, tuple.Item1, tuple.Item2);
+          }
+        }
+      });
+    }
+
+    private List<Tuple<int, int>> GetStartAndEndingsForInterpolation(KeyValuePair<int, IList<int>> column) {
+      var startEndings = new List<Tuple<int, int>>();
+      var rowIndices = column.Value.OrderBy(x => x).ToList();
+      var count = rowIndices.Count;
+      int start = int.MinValue;
+      for (int i = 0; i < count; ++i) {
+        if (start == int.MinValue) {
+          start = IndexOfPrevPresentValue(column.Key, rowIndices[i]);
+        }
+        if (i + 1 == count || (i + 1 < count && rowIndices[i + 1] - rowIndices[i] > 1)) {
+          int next = IndexOfNextPresentValue(column.Key, rowIndices[i]);
+          if (start > 0 && next < PreprocessingData.Rows) {
+            startEndings.Add(new Tuple<int, int>(start, next));
+          }
+          start = int.MinValue;
+        }
+      }
+      return startEndings;
+    }
+
+    private void Interpolate(KeyValuePair<int, IList<int>> column, int prevIndex, int nextIndex) {
+      int valuesToInterpolate = nextIndex - prevIndex;
+
+      if (PreprocessingData.VariableHasType<double>(column.Key)) {
+        double prev = PreprocessingData.GetCell<double>(column.Key, prevIndex);
+        double next = PreprocessingData.GetCell<double>(column.Key, nextIndex);
+        double interpolationStep = (next - prev) / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          double interpolated = prev + (interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<double>(column.Key, i, interpolated);
+        }
+      } else if (PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+        DateTime prev = PreprocessingData.GetCell<DateTime>(column.Key, prevIndex);
+        DateTime next = PreprocessingData.GetCell<DateTime>(column.Key, nextIndex);
+        double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          DateTime interpolated = prev.AddSeconds(interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<DateTime>(column.Key, i, interpolated);
+        }
+      }
+    }
+
+    private int IndexOfPrevPresentValue(int columnIndex, int start) {
+      int offset = start - 1;
+      while (offset >= 0 && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset--;
+      }
+
+      return offset;
+    }
+
+    private int IndexOfNextPresentValue(int columnIndex, int start) {
+      int offset = start + 1;
+      while (offset < PreprocessingData.Rows && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset++;
+      }
+
+      return offset;
+    }
+
+    public void Shuffle(bool shuffleRangesSeparately) {
+      var random = new FastRandom();
+
+      if (shuffleRangesSeparately) {
+        var ranges = new[] { PreprocessingData.TestPartition, PreprocessingData.TrainingPartition };
+        PreprocessingData.InTransaction(() => {
+          // process all given ranges - e.g. TrainingPartition, TestPartition
+          foreach (IntRange range in ranges) {
+            var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
+            var shuffledIndices = Enumerable.Range(range.Start, range.Size).Shuffle(random).ToArray();
+            for (int i = range.Start, j = 0; i < range.End; i++, j++)
+              indices[i] = shuffledIndices[j];
+
+            ReOrderToIndices(indices);
+          }
+        });
+
+      } else {
+        PreprocessingData.InTransaction(() => {
+          var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
+          indices.ShuffleInPlace(random);
+          ReOrderToIndices(indices);
+        });
+      }
+    }
+
+    public void ReOrderToIndices(int[] indices) {
+      PreprocessingData.InTransaction(() => {
+        for (int i = 0; i < PreprocessingData.Columns; ++i) {
+          if (PreprocessingData.VariableHasType<double>(i))
+            ReOrderToIndices<double>(i, indices);
+          else if (PreprocessingData.VariableHasType<string>(i))
+            ReOrderToIndices<string>(i, indices);
+          else if (PreprocessingData.VariableHasType<DateTime>(i))
+            ReOrderToIndices<DateTime>(i, indices);
+        }
+      });
+    }
+
+    private void ReOrderToIndices<T>(int columnIndex, int[] indices) {
+      var originalData = new List<T>(PreprocessingData.GetValues<T>(columnIndex));
+      if (indices.Length != originalData.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+
+      for (int i = 0; i < indices.Length; i++) {
+        T newValue = originalData[indices[i]];
+        PreprocessingData.SetCell<T>(columnIndex, i, newValue);
+      }
+    }
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/FilterContent.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections.Generic;
 using System.Drawing;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.DataPreprocessing.Filter;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Filter", "Represents the filter grid.")]
-  public class FilterContent : Item, IViewShortcut {
+  [StorableClass]
+  public class FilterContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Filter; }
     }
-
-    public FilterLogic FilterLogic { get; private set; }
-
+    [Storable]
     public ICheckedItemCollection<IFilter> Filters { get; private set; }
 
+    [Storable]
     public bool IsAndCombination { get; set; }
 
-    public FilterContent(FilterLogic filterLogic) {
+    public IEnumerable<IFilter> ActiveFilters {
+      get { return Filters.Where(f => f.Active && f.ConstraintData != null); }
+    }
+
+    public bool[] GetRemainingRows() {
+      var remainingRows = new bool[PreprocessingData.Rows];
+      if (ActiveFilters.Any()) {
+        var filterResults = ActiveFilters.Select(f => f.Check()).ToList();
+        var rowFilterResults = new bool[filterResults.Count];
+        for (int row = 0; row < remainingRows.Length; row++) {
+          for (int i = 0; i < filterResults.Count; i++)
+            rowFilterResults[i] = filterResults[i][row];
+
+          remainingRows[row] = IsAndCombination
+            ? rowFilterResults.All(x => x)
+            : rowFilterResults.Any(x => x);
+        }
+      } else {
+        // if not filters active => all rows are remaining
+        for (int i = 0; i < remainingRows.Length; i++)
+          remainingRows[i] = true;
+      }
+      return remainingRows;
+    }
+
+    #region Constructor, Cloning & Persistence
+    public FilterContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
       Filters = new CheckedItemCollection<IFilter>();
       IsAndCombination = true;
-      FilterLogic = filterLogic;
     }
 
-    protected FilterContent(FilterContent content, Cloner cloner)
-      : base(content, cloner) {
+    protected FilterContent(FilterContent original, Cloner cloner)
+      : base(original, cloner) {
+      Filters = cloner.Clone(original.Filters);
+      IsAndCombination = original.IsAndCombination;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new FilterContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected FilterContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/HistogramContent.cs

@@ -26 +26 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Histogram", "Represents the histogram grid.")]
+  [StorableClass]
   public class HistogramContent : PreprocessingChartContent {
     public static new Image StaticItemImage {
@@ -34 +36 @@
     }
 
+    [Storable]
     public string GroupingVariableName { get; set; }
 
+    [Storable]
     public int Bins { get; set; }
+    [Storable]
     public bool ExactBins { get; set; }
 
+    [Storable]
     public LegendOrder Order { get; set; }
 
+    #region Constructor, Cloning & Persistence
     public HistogramContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
@@ -47 +54 @@
     }
 
-    public HistogramContent(HistogramContent content, Cloner cloner)
-      : base(content, cloner) {
+    public HistogramContent(HistogramContent original, Cloner cloner)
+      : base(original, cloner) {
+      GroupingVariableName = original.GroupingVariableName;
+      Bins = original.Bins;
+      ExactBins = original.ExactBins;
+      Order = original.Order;
     }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new HistogramContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected HistogramContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public static DataTable CreateHistogram(IFilteredPreprocessingData preprocessingData, string variableName, string groupingVariableName, DataTableVisualProperties.DataTableHistogramAggregation aggregation, LegendOrder legendOrder = LegendOrder.Alphabetically) {

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/LineChartContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Line Chart", "Represents the line chart grid.")]
+  [StorableClass]
   public class LineChartContent : PreprocessingChartContent {
-    public bool AllInOneMode { get; set; }
-
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Performance; }
     }
 
+    [Storable]
+    public bool AllInOneMode { get; set; }
+
+
+    #region Constructor, Cloning & Persistence
     public LineChartContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
@@ -39 +43 @@
     }
 
-    public LineChartContent(LineChartContent content, Cloner cloner)
-      : base(content, cloner) {
-      this.AllInOneMode = content.AllInOneMode;
+    public LineChartContent(LineChartContent original, Cloner cloner)
+      : base(original, cloner) {
+      AllInOneMode = original.AllInOneMode;
     }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new LineChartContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected LineChartContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs

@@ -20 +20 @@
 #endregion
 
+using System;
+using System.Collections.Generic;
 using System.Drawing;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Manipulation", "Represents the available manipulations on a data set.")]
-  public class ManipulationContent : Item, IViewShortcut {
+  [StorableClass]
+  public class ManipulationContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Method; }
     }
 
-    public ManipulationLogic ManipulationLogic { get; private set; }
-    public SearchLogic SearchLogic { get; private set; }
-    public FilterLogic FilterLogic { get; private set; }
-
-    public ManipulationContent(ManipulationLogic manipulationLogic, SearchLogic searchLogic, FilterLogic filterLogic) {
-      ManipulationLogic = manipulationLogic;
-      SearchLogic = searchLogic;
-      FilterLogic = filterLogic;
+    #region Constructor, Cloning & Persistence
+    public ManipulationContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
     }
 
-    public ManipulationContent(ManipulationContent content, Cloner cloner) : base(content, cloner) { }
-
+    public ManipulationContent(ManipulationContent original, Cloner cloner) :
+      base(original, cloner) {
+    }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new ManipulationContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected ManipulationContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
+
+    public List<int> RowsWithMissingValuesGreater(double percent) {
+      List<int> rows = new List<int>();
+
+      for (int i = 0; i < PreprocessingData.Rows; ++i) {
+        int missingCount = PreprocessingData.GetRowMissingValueCount(i);
+        if (100f / PreprocessingData.Columns * missingCount > percent) {
+          rows.Add(i);
+        }
+      }
+
+      return rows;
+    }
+
+    public List<int> ColumnsWithMissingValuesGreater(double percent) {
+      List<int> columns = new List<int>();
+      for (int i = 0; i < PreprocessingData.Columns; ++i) {
+        int missingCount = PreprocessingData.GetMissingValueCount(i);
+        if (100f / PreprocessingData.Rows * missingCount > percent) {
+          columns.Add(i);
+        }
+      }
+
+      return columns;
+    }
+
+    public List<int> ColumnsWithVarianceSmaller(double variance) {
+      List<int> columns = new List<int>();
+      for (int i = 0; i < PreprocessingData.Columns; ++i) {
+        if (PreprocessingData.VariableHasType<double>(i)) {
+          double columnVariance = PreprocessingData.GetVariance<double>(i);
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        } else if (PreprocessingData.VariableHasType<DateTime>(i)) {
+          double columnVariance = (double)PreprocessingData.GetVariance<DateTime>(i).Ticks / TimeSpan.TicksPerSecond;
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        }
+      }
+      return columns;
+    }
+
+    public void DeleteRowsWithMissingValuesGreater(double percent) {
+      DeleteRows(RowsWithMissingValuesGreater(percent));
+    }
+
+    public void DeleteColumnsWithMissingValuesGreater(double percent) {
+      DeleteColumns(ColumnsWithMissingValuesGreater(percent));
+    }
+
+    public void DeleteColumnsWithVarianceSmaller(double variance) {
+      DeleteColumns(ColumnsWithVarianceSmaller(variance));
+    }
+
+    private void DeleteRows(List<int> rows) {
+      PreprocessingData.InTransaction(() => {
+        foreach (int row in rows.OrderByDescending(x => x)) {
+          PreprocessingData.DeleteRow(row);
+        }
+      });
+    }
+
+    private void DeleteColumns(List<int> columns) {
+      PreprocessingData.InTransaction(() => {
+        foreach (int column in columns.OrderByDescending(x => x)) {
+          PreprocessingData.DeleteColumn(column);
+        }
+      });
+    }
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/MultiScatterPlotContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Multi Scatter Plot", "Represents a multi scatter plot.")]
+  [StorableClass]
   public class MultiScatterPlotContent : ScatterPlotContent {
     public static new Image StaticItemImage {
@@ -32 +33 @@
     }
 
+    #region Constructor, Cloning & Persistence
     public MultiScatterPlotContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
     }
 
-    public MultiScatterPlotContent(MultiScatterPlotContent content, Cloner cloner)
-      : base(content, cloner) {
+    public MultiScatterPlotContent(MultiScatterPlotContent original, Cloner cloner)
+      : base(original, cloner) {
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new MultiScatterPlotContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected MultiScatterPlotContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingChartContent.cs

@@ -29 +29 @@
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("PreprocessingChart", "Represents a preprocessing chart.")]
-  public class PreprocessingChartContent : Item, IViewShortcut {
+  [StorableClass]
+  public class PreprocessingChartContent : PreprocessingContent, IViewShortcut {
     public enum LegendOrder {
       Alphabetically,
@@ -42 +44 @@
     }
 
-    private ICheckedItemList<StringValue> variableItemList = null;
+    [Storable]
+    private ICheckedItemList<StringValue> variableItemList;
     public ICheckedItemList<StringValue> VariableItemList {
       get {
         if (variableItemList == null)
           variableItemList = CreateVariableItemList(PreprocessingData);
-        return this.variableItemList;
+        return variableItemList;
       }
     }
 
-    public IFilteredPreprocessingData PreprocessingData { get; private set; }
     public event DataPreprocessingChangedEventHandler Changed {
       add { PreprocessingData.Changed += value; }
@@ -57 +59 @@
     }
 
-    public PreprocessingChartContent(IFilteredPreprocessingData preprocessingData) {
-      PreprocessingData = preprocessingData;
+    #region Constructor, Cloning & Persistence
+    public PreprocessingChartContent(IFilteredPreprocessingData preprocessingData)
+       : base(preprocessingData) {
     }
 
-    public PreprocessingChartContent(PreprocessingChartContent content, Cloner cloner)
-      : base(content, cloner) {
-      this.PreprocessingData = content.PreprocessingData;
-      this.variableItemList = cloner.Clone<ICheckedItemList<StringValue>>(variableItemList);
+    public PreprocessingChartContent(PreprocessingChartContent original, Cloner cloner)
+      : base(original, cloner) {
+      variableItemList = cloner.Clone(original.variableItemList);
     }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new PreprocessingChartContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected PreprocessingChartContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public DataRow CreateDataRow(string variableName, DataRowVisualProperties.DataRowChartType chartType) {
@@ -75 +82 @@
 
     public static DataRow CreateDataRow(IFilteredPreprocessingData preprocessingData, string variableName, DataRowVisualProperties.DataRowChartType chartType) {
-      IList<double> values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
-      DataRow row = new DataRow(variableName, "", values);
-      row.VisualProperties.ChartType = chartType;
+      var values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
+      var row = new DataRow(variableName, "", values) {
+        VisualProperties = {
+          ChartType = chartType,
+          StartIndexZero = true
+        }
+      };
       return row;
     }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/ScatterPlotContent.cs

@@ -25 +25 @@
 using HeuristicLab.Analysis;
 using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Visualization.ChartControlsExtensions;
 
 namespace HeuristicLab.DataPreprocessing {
-
+  [Item("ScatterPlotContent", "")]
+  [StorableClass]
   public abstract class ScatterPlotContent : PreprocessingChartContent {
+    [Storable]
     public string GroupingVariable { get; set; }
 
+    #region Constructor, Cloning & Persistence
     protected ScatterPlotContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
     }
 
-    protected ScatterPlotContent(ScatterPlotContent content, Cloner cloner)
-      : base(content, cloner) {
+    protected ScatterPlotContent(ScatterPlotContent original, Cloner cloner)
+      : base(original, cloner) {
+      GroupingVariable = original.GroupingVariable;
     }
+
+    [StorableConstructor]
+    protected ScatterPlotContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public static ScatterPlot CreateScatterPlot(IFilteredPreprocessingData preprocessingData, string variableNameX, string variableNameY, string variableNameGroup = "-", LegendOrder legendOrder = LegendOrder.Alphabetically) {

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/SingleScatterPlotContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Scatter Plot", "Represents a scatter plot.")]
+  [StorableClass]
   public class SingleScatterPlotContent : ScatterPlotContent {
     public static new Image StaticItemImage {
@@ -32 +33 @@
     }
 
+    [Storable]
     public string SelectedXVariable { get; set; }
+    [Storable]
     public string SelectedYVariable { get; set; }
 
+    #region Constructor, Cloning & Persistence
     public SingleScatterPlotContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
     }
 
-    public SingleScatterPlotContent(SingleScatterPlotContent content, Cloner cloner)
-      : base(content, cloner) {
-      this.SelectedXVariable = content.SelectedXVariable;
-      this.SelectedYVariable = content.SelectedYVariable;
-      this.GroupingVariable = content.GroupingVariable;
+    public SingleScatterPlotContent(SingleScatterPlotContent original, Cloner cloner)
+      : base(original, cloner) {
+      SelectedXVariable = original.SelectedXVariable;
+      SelectedYVariable = original.SelectedYVariable;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new SingleScatterPlotContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected SingleScatterPlotContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/StatisticsContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Statistics", "Represents the statistics grid.")]
-  public class StatisticsContent : Item, IViewShortcut {
+  [StorableClass]
+  public class StatisticsContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Object; }
     }
 
-    public ITransactionalPreprocessingData PreprocessingData { get; private set; }
-    public StatisticsLogic StatisticsLogic { get; private set; }
-
-    public StatisticsContent(ITransactionalPreprocessingData preProcessingData, StatisticsLogic statisticsLogic) {
-      PreprocessingData = preProcessingData;
-      StatisticsLogic = statisticsLogic;
+    #region Constructor, Cloning & Persistence
+    public StatisticsContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
     }
 
-    public StatisticsContent(StatisticsContent content, Cloner cloner)
-      : base(content, cloner) {
+    public StatisticsContent(StatisticsContent original, Cloner cloner)
+      : base(original, cloner) {
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new StatisticsContent(this, cloner);
     }
 
+    [StorableConstructor]
+    protected StatisticsContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
+
     public event DataPreprocessingChangedEventHandler Changed {
-      add { StatisticsLogic.Changed += value; }
-      remove { StatisticsLogic.Changed -= value; }
+      add { PreprocessingData.Changed += value; }
+      remove { PreprocessingData.Changed -= value; }
     }
   }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/TransformationContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Transformation", "Represents the transformation grid.")]
-  public class TransformationContent : Item, IViewShortcut {
+  [StorableClass]
+  public class TransformationContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Method; }
     }
 
-    public IPreprocessingData Data { get; private set; }
-    public FilterLogic FilterLogic { get; private set; }
-
+    [Storable]
     public ICheckedItemList<ITransformation> CheckedTransformationList { get; private set; }
 
-    public TransformationContent(IPreprocessingData data, FilterLogic filterLogic) {
-      Data = data;
+    #region Constructor, Cloning & Persistence
+    public TransformationContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
       CheckedTransformationList = new CheckedItemList<ITransformation>();
-      FilterLogic = filterLogic;
     }
 
     public TransformationContent(TransformationContent original, Cloner cloner)
       : base(original, cloner) {
-      Data = original.Data;
-      CheckedTransformationList = new CheckedItemList<ITransformation>(original.CheckedTransformationList);
+      CheckedTransformationList = cloner.Clone(original.CheckedTransformationList);
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new TransformationContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected TransformationContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Data/FilteredPreprocessingData.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.DataPreprocessing {
-  public class FilteredPreprocessingData : NamedItem, IFilteredPreprocessingData {
-    private readonly ITransactionalPreprocessingData originalData;
-    private ITransactionalPreprocessingData filteredData;
-
+  [Item("FilteredPreprocessingData", "Represents filtered data used for preprocessing.")]
+  [StorableClass]
+  public sealed class FilteredPreprocessingData : NamedItem, IFilteredPreprocessingData {
+
+    [Storable]
+    private readonly IPreprocessingData originalData;
+    [Storable]
+    private IPreprocessingData filteredData;
+
+    public IPreprocessingData ActiveData {
+      get { return IsFiltered ? filteredData : originalData; }
+    }
+
+    #region Constructor, Cloning & Persistence
+    public FilteredPreprocessingData(IPreprocessingData preprocessingData)
+      : base() {
+      originalData = preprocessingData;
+      filteredData = null;
+    }
+
+    private FilteredPreprocessingData(FilteredPreprocessingData original, Cloner cloner)
+      : base(original, cloner) {
+      originalData = original.originalData;
+      filteredData = original.filteredData;
+    }
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new FilteredPreprocessingData(this, cloner);
+    }
+
+    [StorableConstructor]
+    private FilteredPreprocessingData(bool deserializing)
+      : base(deserializing) { }
+    #endregion
+
+    #region Cells
+    public bool IsCellEmpty(int columnIndex, int rowIndex) {
+      return ActiveData.IsCellEmpty(columnIndex, rowIndex);
+    }
+
+    public T GetCell<T>(int columnIndex, int rowIndex) {
+      return ActiveData.GetCell<T>(columnIndex, rowIndex);
+    }
+
+    public void SetCell<T>(int columnIndex, int rowIndex, T value) {
+      if (IsFiltered)
+        throw new InvalidOperationException("SetValues not possible while data is filtered");
+      originalData.SetCell<T>(columnIndex, rowIndex, value);
+    }
+
+    public string GetCellAsString(int columnIndex, int rowIndex) {
+      return ActiveData.GetCellAsString(columnIndex, rowIndex);
+    }
+
+    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
+      return ActiveData.GetValues<T>(columnIndex, considerSelection);
+    }
+
+    public void SetValues<T>(int columnIndex, IList<T> values) {
+      if (IsFiltered)
+        throw new InvalidOperationException("SetValues not possible while data is filtered");
+
+      originalData.SetValues<T>(columnIndex, values);
+    }
+
+    public bool SetValue(string value, int columnIndex, int rowIndex) {
+      if (IsFiltered)
+        throw new InvalidOperationException("SetValue not possible while data is filtered");
+      return originalData.SetValue(value, columnIndex, rowIndex);
+    }
+
+    public int Columns {
+      get { return ActiveData.Columns; }
+    }
+
+    public int Rows {
+      get { return ActiveData.Rows; }
+    }
+    #endregion
+
+    #region Rows
+    public void InsertRow(int rowIndex) {
+      if (IsFiltered)
+        throw new InvalidOperationException("InsertRow not possible while data is filtered");
+
+      originalData.InsertRow(rowIndex);
+    }
+
+    public void DeleteRow(int rowIndex) {
+      if (IsFiltered)
+        throw new InvalidOperationException("DeleteRow not possible while data is filtered");
+
+      originalData.DeleteRow(rowIndex);
+    }
+
+    public void DeleteRowsWithIndices(IEnumerable<int> rows) {
+      if (IsFiltered)
+        throw new InvalidOperationException("DeleteRowsWithIndices not possible while data is filtered");
+
+      originalData.DeleteRowsWithIndices(rows);
+    }
+
+    public void InsertColumn<T>(string variableName, int columnIndex) {
+      if (IsFiltered)
+        throw new InvalidOperationException("InsertColumn not possible while data is filtered");
+
+      originalData.InsertColumn<T>(variableName, columnIndex);
+    }
+
+    public void DeleteColumn(int columnIndex) {
+      if (IsFiltered)
+        throw new InvalidOperationException("DeleteColumn not possible while data is filtered");
+      originalData.DeleteColumn(columnIndex);
+    }
+
+    public void RenameColumn(int columnIndex, string name) {
+      if (IsFiltered)
+        throw new InvalidOperationException("RenameColumn not possible while data is filtered");
+      originalData.RenameColumn(columnIndex, name);
+    }
+
+    public void RenameColumns(IList<string> names) {
+      if (IsFiltered)
+        throw new InvalidOperationException("RenameColumns not possible while data is filtered");
+      originalData.RenameColumns(names);
+    }
+
+    public bool AreAllStringColumns(IEnumerable<int> columnIndices) {
+      return originalData.AreAllStringColumns(columnIndices);
+    }
+    #endregion
+
+    #region Variables
+    public IEnumerable<string> VariableNames {
+      get { return ActiveData.VariableNames; }
+    }
+    public IEnumerable<string> GetDoubleVariableNames() {
+      return originalData.GetDoubleVariableNames();
+    }
+    public string GetVariableName(int columnIndex) {
+      return ActiveData.GetVariableName(columnIndex);
+    }
+
+    public int GetColumnIndex(string variableName) {
+      return ActiveData.GetColumnIndex(variableName);
+    }
+
+    public bool VariableHasType<T>(int columnIndex) {
+      return originalData.VariableHasType<T>(columnIndex);
+    }
+
+    public Type GetVariableType(int columnIndex) {
+      return ActiveData.GetVariableType(columnIndex);
+    }
+
+    public IList<string> InputVariables {
+      get { return ActiveData.InputVariables; }
+    }
+
+    public string TargetVariable {
+      get { return ActiveData.TargetVariable; }
+    } // optional
+    #endregion
+
+    #region Partitions
     public IntRange TrainingPartition {
       get { return originalData.TrainingPartition; }
@@ -39 +201 @@
       get { return originalData.TestPartition; }
     }
-
+    #endregion
+
+    #region Transformations
     public IList<ITransformation> Transformations {
       get { return originalData.Transformations; }
     }
-
-    public IEnumerable<string> VariableNames {
-      get { return ActiveData.VariableNames; }
-    }
-
-    public IList<string> InputVariables { get { return ActiveData.InputVariables; } }
-    public string TargetVariable { get { return ActiveData.TargetVariable; } } // optional
-
+    #endregion
+
+    #region Validation
+    public bool Validate(string value, out string errorMessage, int columnIndex) {
+      return originalData.Validate(value, out errorMessage, columnIndex);
+    }
+    #endregion
+
+    #region Import & Export
+    public void Import(IDataAnalysisProblemData problemData) {
+      if (IsFiltered)
+        throw new InvalidOperationException("Import not possible while data is filtered");
+      originalData.Import(problemData);
+    }
+
+    public Dataset ExportToDataset() {
+      return originalData.ExportToDataset();
+    }
+    #endregion
+
+    #region Selection
     public IDictionary<int, IList<int>> Selection {
       get { return originalData.Selection; }
@@ -56 +233 @@
     }
 
-    public int Columns {
-      get { return ActiveData.Columns; }
-    }
-
-    public int Rows {
-      get { return ActiveData.Rows; }
-    }
-
-    public ITransactionalPreprocessingData ActiveData {
-      get { return IsFiltered ? filteredData : originalData; }
+    public void ClearSelection() {
+      originalData.ClearSelection();
+    }
+
+    public event EventHandler SelectionChanged {
+      add { originalData.SelectionChanged += value; }
+      remove { originalData.SelectionChanged -= value; }
+    }
+    #endregion
+
+    #region Transactions
+    public event DataPreprocessingChangedEventHandler Changed {
+      add { originalData.Changed += value; }
+      remove { originalData.Changed -= value; }
     }
 
@@ -72 +253 @@
     }
 
-    public bool IsFiltered {
-      get { return filteredData != null; }
-    }
-
-
-    public FilteredPreprocessingData(ITransactionalPreprocessingData preporcessingData)
-      : base() {
-      originalData = preporcessingData;
-      filteredData = null;
-    }
-
-    protected FilteredPreprocessingData(FilteredPreprocessingData original, Cloner cloner)
-      : base(original, cloner) {
-      originalData = original.originalData;
-      filteredData = original.filteredData;
-    }
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new FilteredPreprocessingData(this, cloner);
-    }
-
-    public T GetCell<T>(int columnIndex, int rowIndex) {
-      return ActiveData.GetCell<T>(columnIndex, rowIndex);
-    }
-
-    public void SetCell<T>(int columnIndex, int rowIndex, T value) {
-      if (IsFiltered)
-        throw new InvalidOperationException("SetValues not possible while data is filtered");
-      originalData.SetCell<T>(columnIndex, rowIndex, value);
-    }
-
-    public string GetCellAsString(int columnIndex, int rowIndex) {
-      return ActiveData.GetCellAsString(columnIndex, rowIndex);
-    }
-
-    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
-      return ActiveData.GetValues<T>(columnIndex, considerSelection);
-    }
-
-    public void SetValues<T>(int columnIndex, IList<T> values) {
-      if (IsFiltered)
-        throw new InvalidOperationException("SetValues not possible while data is filtered");
-
-      originalData.SetValues<T>(columnIndex, values);
-    }
-
-    public void InsertRow(int rowIndex) {
-      if (IsFiltered)
-        throw new InvalidOperationException("InsertRow not possible while data is filtered");
-
-      originalData.InsertRow(rowIndex);
-    }
-
-    public void DeleteRow(int rowIndex) {
-      if (IsFiltered)
-        throw new InvalidOperationException("DeleteRow not possible while data is filtered");
-
-      originalData.DeleteRow(rowIndex);
-    }
-
-    public void InsertColumn<T>(string variableName, int columnIndex) {
-      if (IsFiltered)
-        throw new InvalidOperationException("InsertColumn not possible while data is filtered");
-
-      originalData.InsertColumn<T>(variableName, columnIndex);
-    }
-
-    public void DeleteColumn(int columnIndex) {
-      if (IsFiltered)
-        throw new InvalidOperationException("DeleteColumn not possible while data is filtered");
-      originalData.DeleteColumn(columnIndex);
-    }
-
-    public void RenameColumn(int columnIndex, string name) {
-      if (IsFiltered)
-        throw new InvalidOperationException("RenameColumn not possible while data is filtered");
-      originalData.RenameColumn(columnIndex, name);
-    }
-
-    public void RenameColumns(IList<string> names) {
-      if (IsFiltered)
-        throw new InvalidOperationException("RenameColumns not possible while data is filtered");
-      originalData.RenameColumns(names);
-    }
-
-    public string GetVariableName(int columnIndex) {
-      return ActiveData.GetVariableName(columnIndex);
-    }
-
-    public int GetColumnIndex(string variableName) {
-      return ActiveData.GetColumnIndex(variableName);
-    }
-
-    public bool VariableHasType<T>(int columnIndex) {
-      return originalData.VariableHasType<T>(columnIndex);
-    }
-
-    public Dataset ExportToDataset() {
-      return originalData.ExportToDataset();
-    }
-
-    public void SetFilter(bool[] rowFilters) {
-      filteredData = (ITransactionalPreprocessingData)originalData.Clone();
+    public void Undo() {
+      if (IsFiltered)
+        throw new InvalidOperationException("Undo not possible while data is filtered");
+
+      originalData.Undo();
+    }
+
+    public void InTransaction(Action action, DataPreprocessingChangedEventType type = DataPreprocessingChangedEventType.Any) {
+      if (IsFiltered)
+        throw new InvalidOperationException("Transaction not possible while data is filtered");
+      originalData.InTransaction(action, type);
+    }
+
+    public void BeginTransaction(DataPreprocessingChangedEventType type) {
+      if (IsFiltered)
+        throw new InvalidOperationException("Transaction not possible while data is filtered");
+      originalData.BeginTransaction(type);
+    }
+
+    public void EndTransaction() {
+      originalData.EndTransaction();
+    }
+    #endregion
+
+    #region Statistics
+    public T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      return ActiveData.GetMin<T>(columnIndex, considerSelection, emptyValue);
+    }
+    public T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      return ActiveData.GetMax<T>(columnIndex, considerSelection, emptyValue);
+    }
+    public T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      return ActiveData.GetMean<T>(columnIndex, considerSelection, emptyValue);
+    }
+    public T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+      return ActiveData.GetMean<T>(columnIndex, considerSelection, emptyValue);
+    }
+    public T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T> {
+      return ActiveData.GetMode<T>(columnIndex, considerSelection, emptyValue);
+    }
+    public T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      return ActiveData.GetStandardDeviation<T>(columnIndex, considerSelection, emptyValue);
+    }
+    public T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      return ActiveData.GetVariance<T>(columnIndex, considerSelection, emptyValue);
+    }
+    public T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+      return ActiveData.GetQuantile<T>(alpha, columnIndex, considerSelection, emptyValue);
+    }
+    public int GetDistinctValues<T>(int columnIndex, bool considerSelection = false) {
+      return ActiveData.GetDistinctValues<T>(columnIndex, considerSelection);
+    }
+
+    public int GetMissingValueCount() {
+      return ActiveData.GetMissingValueCount();
+    }
+    public int GetMissingValueCount(int columnIndex) {
+      return ActiveData.GetMissingValueCount(columnIndex);
+    }
+    public int GetRowMissingValueCount(int rowIndex) {
+      return ActiveData.GetRowMissingValueCount(rowIndex);
+    }
+    #endregion
+
+    #region Filters
+    public void SetFilter(bool[] remainingRows) {
+      filteredData = (IPreprocessingData)originalData.Clone();
       filteredData.InTransaction(() => {
-        for (int row = (rowFilters.Length - 1); row >= 0; --row) {
-          if (rowFilters[row]) {
-            filteredData.DeleteRow(row);
+        var remainingIndices = Enumerable.Range(0, remainingRows.Length).Where(x => remainingRows[x]);
+
+        foreach (var v in filteredData.VariableNames) {
+          var ci = filteredData.GetColumnIndex(v);
+          if (filteredData.VariableHasType<double>(ci)) {
+            var values = filteredData.GetValues<double>(ci);
+            var filteredValues = remainingIndices.Select(x => values[x]).ToList();
+            filteredData.SetValues(ci, filteredValues);
+          } else if (filteredData.VariableHasType<DateTime>(ci)) {
+            var values = filteredData.GetValues<DateTime>(ci);
+            var filteredValues = remainingIndices.Select(x => values[x]).ToList();
+            filteredData.SetValues(ci, filteredValues);
+          } else if (filteredData.VariableHasType<string>(ci)) {
+            var values = filteredData.GetValues<string>(ci);
+            var filteredValues = remainingIndices.Select(x => values[x]).ToList();
+            filteredData.SetValues(ci, filteredValues);
           }
         }
@@ -206 +365 @@
     }
 
+    public bool IsFiltered {
+      get { return filteredData != null; }
+    }
+
+    public event EventHandler FilterChanged;
+
     private void OnFilterChanged() {
       if (FilterChanged != null) {
@@ -211 +376 @@
       }
     }
-
-    public event DataPreprocessingChangedEventHandler Changed {
-      add { originalData.Changed += value; }
-      remove { originalData.Changed -= value; }
-    }
-
-    public bool SetValue(string value, int columnIndex, int rowIndex) {
-      if (IsFiltered)
-        throw new InvalidOperationException("SetValue not possible while data is filtered");
-      return originalData.SetValue(value, columnIndex, rowIndex);
-    }
-
-    public bool AreAllStringColumns(IEnumerable<int> columnIndices) {
-      return originalData.AreAllStringColumns(columnIndices);
-    }
-
-    public void DeleteRowsWithIndices(IEnumerable<int> rows) {
-      if (IsFiltered)
-        throw new InvalidOperationException("DeleteRowsWithIndices not possible while data is filtered");
-
-      originalData.DeleteRowsWithIndices(rows);
-    }
-
-    public void Undo() {
-      if (IsFiltered)
-        throw new InvalidOperationException("Undo not possible while data is filtered");
-
-      originalData.Undo();
-    }
-
-    public void InTransaction(Action action, DataPreprocessingChangedEventType type = DataPreprocessingChangedEventType.Any) {
-      if (IsFiltered)
-        throw new InvalidOperationException("Transaction not possible while data is filtered");
-      originalData.InTransaction(action, type);
-    }
-
-    public void BeginTransaction(DataPreprocessingChangedEventType type) {
-      if (IsFiltered)
-        throw new InvalidOperationException("Transaction not possible while data is filtered");
-      originalData.BeginTransaction(type);
-    }
-
-    public void EndTransaction() {
-      originalData.EndTransaction();
-    }
-
-    public IEnumerable<string> GetDoubleVariableNames() {
-      return originalData.GetDoubleVariableNames();
-    }
-
-    public void ClearSelection() {
-      originalData.ClearSelection();
-    }
-
-    public event EventHandler SelectionChanged {
-      add { originalData.SelectionChanged += value; }
-      remove { originalData.SelectionChanged -= value; }
-    }
-
-    #region IPreprocessingData Members
-    public bool Validate(string value, out string errorMessage, int columnIndex) {
-      return originalData.Validate(value, out errorMessage, columnIndex);
-    }
-
-    public event EventHandler FilterChanged;
     #endregion
   }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Data/IFilteredPreprocessingData.cs

@@ -23 +23 @@
 
 namespace HeuristicLab.DataPreprocessing {
-  public interface IFilteredPreprocessingData : ITransactionalPreprocessingData {
-    void SetFilter(bool[] rowFilters);
+  public interface IFilteredPreprocessingData : IPreprocessingData {
+    #region Filters
+    void SetFilter(bool[] remainingRows);
     void PersistFilter();
     void ResetFilter();
@@ -30 +31 @@
 
     event EventHandler FilterChanged;
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Data/IPreprocessingData.cs

@@ -28 +28 @@
 namespace HeuristicLab.DataPreprocessing {
   public interface IPreprocessingData : INamedItem {
+    #region Cells
+    bool IsCellEmpty(int columnIndex, int rowIndex);
     T GetCell<T>(int columnIndex, int rowIndex);
 
@@ -39 +41 @@
     bool SetValue(string value, int columnIndex, int rowIndex);
 
+    int Columns { get; }
+    int Rows { get; }
+    #endregion
+
+    #region Rows
     void InsertRow(int rowIndex);
     void DeleteRow(int rowIndex);
@@ -50 +57 @@
 
     bool AreAllStringColumns(IEnumerable<int> columnIndices);
-    bool Validate(string value, out string errorMessage, int columnIndex);
+    #endregion
 
-    IntRange TrainingPartition { get; }
-    IntRange TestPartition { get; }
-
-    IList<ITransformation> Transformations { get; }
-
+    #region Variables
     IEnumerable<string> VariableNames { get; }
     IEnumerable<string> GetDoubleVariableNames();
@@ -63 +66 @@
 
     bool VariableHasType<T>(int columnIndex);
+    Type GetVariableType(int columnIndex);
 
     IList<string> InputVariables { get; }
     string TargetVariable { get; } // optional
+    #endregion
 
-    int Columns { get; }
-    int Rows { get; }
+    #region Partitions
+    IntRange TrainingPartition { get; }
+    IntRange TestPartition { get; }
+    #endregion
 
+    #region Transformations
+    IList<ITransformation> Transformations { get; }
+    #endregion
+
+    #region Validation
+    bool Validate(string value, out string errorMessage, int columnIndex);
+    #endregion
+
+    #region Import & Export
+    void Import(IDataAnalysisProblemData problemData);
     Dataset ExportToDataset();
+    #endregion
 
+    #region Selection
     IDictionary<int, IList<int>> Selection { get; set; }
     void ClearSelection();
 
     event EventHandler SelectionChanged;
+    #endregion
+
+    #region Transactions
+    event DataPreprocessingChangedEventHandler Changed;
+
+    bool IsUndoAvailable { get; }
+    void Undo();
+    void InTransaction(Action action, DataPreprocessingChangedEventType type = DataPreprocessingChangedEventType.Any);
+    void BeginTransaction(DataPreprocessingChangedEventType type);
+    void EndTransaction();
+    #endregion
+
+    #region Statistics
+    T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
+    T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
+    T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
+    T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T>;
+    T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T>;
+    T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
+    T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T));
+    T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T>;
+    int GetDistinctValues<T>(int columnIndex, bool considerSelection = false);
+
+    int GetMissingValueCount();
+    int GetMissingValueCount(int columnIndex);
+    int GetRowMissingValueCount(int rowIndex);
+    #endregion
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs

@@ -23 +23 @@
 using System.Collections;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 
@@ -32 +34 @@
 
   [Item("PreprocessingData", "Represents data used for preprocessing.")]
-  public abstract class PreprocessingData : NamedItem, IPreprocessingData {
-    public IntRange TrainingPartition { get; set; }
-    public IntRange TestPartition { get; set; }
-
-    public IList<ITransformation> Transformations { get; protected set; }
-
+  [StorableClass]
+  public class PreprocessingData : NamedItem, IPreprocessingData {
+
+    [Storable]
     protected IList<IList> variableValues;
+    [Storable]
     protected IList<string> variableNames;
 
-    public IEnumerable<string> VariableNames {
-      get { return variableNames; }
-    }
-
-    public IEnumerable<string> GetDoubleVariableNames() {
-      var doubleVariableNames = new List<string>();
-      for (int i = 0; i < Columns; ++i) {
-        if (VariableHasType<double>(i)) {
-          doubleVariableNames.Add(variableNames[i]);
-        }
-      }
-      return doubleVariableNames;
-    }
-
-    public IList<string> InputVariables { get; private set; }
-    public string TargetVariable { get; private set; } // optional
-
-    public int Columns {
-      get { return variableNames.Count; }
-    }
-
-    public int Rows {
-      get { return variableValues.Count > 0 ? variableValues[0].Count : 0; }
-    }
-
-    protected IDictionary<int, IList<int>> selection;
-    public IDictionary<int, IList<int>> Selection {
-      get { return selection; }
-      set {
-        selection = value;
-        OnSelectionChanged();
-      }
+    #region Constructor, Cloning & Persistence
+    public PreprocessingData(IDataAnalysisProblemData problemData)
+      : base() {
+      Name = "Preprocessing Data";
+
+      Transformations = new List<ITransformation>();
+      selection = new Dictionary<int, IList<int>>();
+
+      Import(problemData);
+
+      RegisterEventHandler();
     }
 
@@ -88 +68 @@
       RegisterEventHandler();
     }
-
-    protected PreprocessingData(IDataAnalysisProblemData problemData)
-      : base() {
-      Name = "Preprocessing Data";
-
-      Transformations = new List<ITransformation>();
-      selection = new Dictionary<int, IList<int>>();
-
-      Import(problemData);
-
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new PreprocessingData(this, cloner);
+    }
+
+    [StorableConstructor]
+    protected PreprocessingData(bool deserializing)
+      : base(deserializing) { }
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
       RegisterEventHandler();
     }
 
+    private void RegisterEventHandler() {
+      Changed += (s, e) => {
+        switch (e.Type) {
+          case DataPreprocessingChangedEventType.DeleteRow:
+          case DataPreprocessingChangedEventType.Any:
+          case DataPreprocessingChangedEventType.Transformation:
+            int maxRowIndex = Math.Max(0, Rows);
+            TrainingPartition.Start = Math.Min(TrainingPartition.Start, maxRowIndex);
+            TrainingPartition.End = Math.Min(TrainingPartition.End, maxRowIndex);
+            TestPartition.Start = Math.Min(TestPartition.Start, maxRowIndex);
+            TestPartition.End = Math.Min(TestPartition.End, maxRowIndex);
+            break;
+        }
+      };
+    }
+    #endregion
+
+    #region Cells
+    public bool IsCellEmpty(int columnIndex, int rowIndex) {
+      var value = variableValues[columnIndex][rowIndex];
+      return IsMissingValue(value);
+    }
+
+    public T GetCell<T>(int columnIndex, int rowIndex) {
+      return (T)variableValues[columnIndex][rowIndex];
+    }
+
+    public void SetCell<T>(int columnIndex, int rowIndex, T value) {
+      SaveSnapshot(DataPreprocessingChangedEventType.ChangeItem, columnIndex, rowIndex);
+
+      for (int i = Rows; i <= rowIndex; i++)
+        InsertRow(i);
+      for (int i = Columns; i <= columnIndex; i++)
+        InsertColumn<T>(i.ToString(), i);
+
+      variableValues[columnIndex][rowIndex] = value;
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.ChangeItem, columnIndex, rowIndex);
+    }
+
+    public string GetCellAsString(int columnIndex, int rowIndex) {
+      return variableValues[columnIndex][rowIndex].ToString();
+    }
+
+    public IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
+      if (considerSelection) {
+        var list = new List<T>();
+        foreach (var rowIdx in selection[columnIndex]) {
+          list.Add((T)variableValues[columnIndex][rowIdx]);
+        }
+        return list;
+      } else {
+        return (IList<T>)variableValues[columnIndex];
+      }
+    }
+
+    public void SetValues<T>(int columnIndex, IList<T> values) {
+      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
+      if (VariableHasType<T>(columnIndex)) {
+        variableValues[columnIndex] = (IList)values;
+      } else {
+        throw new ArgumentException("The datatype of column " + columnIndex + " must be of type " + variableValues[columnIndex].GetType().Name + " but was " + typeof(T).Name);
+      }
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
+    }
+
+    public bool SetValue(string value, int columnIndex, int rowIndex) {
+      bool valid = false;
+      if (VariableHasType<double>(columnIndex)) {
+        double val;
+        if (string.IsNullOrWhiteSpace(value)) {
+          val = double.NaN;
+          valid = true;
+        } else {
+          valid = double.TryParse(value, out val);
+        }
+        if (valid)
+          SetCell(columnIndex, rowIndex, val);
+      } else if (VariableHasType<string>(columnIndex)) {
+        valid = value != null;
+        if (valid)
+          SetCell(columnIndex, rowIndex, value);
+      } else if (VariableHasType<DateTime>(columnIndex)) {
+        DateTime date;
+        valid = DateTime.TryParse(value, out date);
+        if (valid)
+          SetCell(columnIndex, rowIndex, date);
+      } else {
+        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
+      }
+
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
+
+      return valid;
+    }
+
+    public int Columns {
+      get { return variableNames.Count; }
+    }
+
+    public int Rows {
+      get { return variableValues.Count > 0 ? variableValues[0].Count : 0; }
+    }
+
+    public static bool IsMissingValue(object value) {
+      if (value is double) return double.IsNaN((double)value);
+      if (value is string) return string.IsNullOrEmpty((string)value);
+      if (value is DateTime) return ((DateTime)value).Equals(DateTime.MinValue);
+      throw new ArgumentException();
+    }
+    #endregion
+
+    #region Rows
+    public void InsertRow(int rowIndex) {
+      SaveSnapshot(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
+      foreach (IList column in variableValues) {
+        Type type = column.GetType().GetGenericArguments()[0];
+        column.Insert(rowIndex, type.IsValueType ? Activator.CreateInstance(type) : null);
+      }
+      if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
+        TrainingPartition.End++;
+        if (TrainingPartition.End <= TestPartition.Start) {
+          TestPartition.Start++;
+          TestPartition.End++;
+        }
+      } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
+        TestPartition.End++;
+        if (TestPartition.End <= TrainingPartition.Start) {
+          TestPartition.Start++;
+          TestPartition.End++;
+        }
+      }
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
+    }
+    public void DeleteRow(int rowIndex) {
+      SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
+      foreach (IList column in variableValues) {
+        column.RemoveAt(rowIndex);
+      }
+      if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
+        TrainingPartition.End--;
+        if (TrainingPartition.End <= TestPartition.Start) {
+          TestPartition.Start--;
+          TestPartition.End--;
+        }
+      } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
+        TestPartition.End--;
+        if (TestPartition.End <= TrainingPartition.Start) {
+          TestPartition.Start--;
+          TestPartition.End--;
+        }
+      }
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
+    }
+    public void DeleteRowsWithIndices(IEnumerable<int> rows) {
+      SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, -1);
+      foreach (int rowIndex in rows.OrderByDescending(x => x)) {
+        foreach (IList column in variableValues) {
+          column.RemoveAt(rowIndex);
+        }
+        if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
+          TrainingPartition.End--;
+          if (TrainingPartition.End <= TestPartition.Start) {
+            TestPartition.Start--;
+            TestPartition.End--;
+          }
+        } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
+          TestPartition.End--;
+          if (TestPartition.End <= TrainingPartition.Start) {
+            TestPartition.Start--;
+            TestPartition.End--;
+          }
+        }
+      }
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, -1);
+    }
+
+    public void InsertColumn<T>(string variableName, int columnIndex) {
+      SaveSnapshot(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
+      variableValues.Insert(columnIndex, new List<T>(Enumerable.Repeat(default(T), Rows)));
+      variableNames.Insert(columnIndex, variableName);
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
+    }
+
+    public void DeleteColumn(int columnIndex) {
+      SaveSnapshot(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
+      variableValues.RemoveAt(columnIndex);
+      variableNames.RemoveAt(columnIndex);
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
+    }
+
+    public void RenameColumn(int columnIndex, string name) {
+      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
+      if (columnIndex < 0 || columnIndex > variableNames.Count)
+        throw new ArgumentOutOfRangeException("columnIndex");
+      variableNames[columnIndex] = name;
+
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
+    }
+
+    public void RenameColumns(IList<string> names) {
+      if (names == null) throw new ArgumentNullException("names");
+      if (names.Count != variableNames.Count) throw new ArgumentException("number of names must match the number of columns.", "names");
+
+      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
+      for (int i = 0; i < names.Count; i++)
+        variableNames[i] = names[i];
+
+      if (!IsInTransaction)
+        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
+    }
+
+    public bool AreAllStringColumns(IEnumerable<int> columnIndices) {
+      return columnIndices.All(x => VariableHasType<string>(x));
+    }
+    #endregion
+
+    #region Variables
+    public IEnumerable<string> VariableNames {
+      get { return variableNames; }
+    }
+
+    public IEnumerable<string> GetDoubleVariableNames() {
+      var doubleVariableNames = new List<string>();
+      for (int i = 0; i < Columns; ++i) {
+        if (VariableHasType<double>(i)) {
+          doubleVariableNames.Add(variableNames[i]);
+        }
+      }
+      return doubleVariableNames;
+    }
+
+    public string GetVariableName(int columnIndex) {
+      return variableNames[columnIndex];
+    }
+
+    public int GetColumnIndex(string variableName) {
+      return variableNames.IndexOf(variableName);
+    }
+
+    public bool VariableHasType<T>(int columnIndex) {
+      return columnIndex >= variableValues.Count || variableValues[columnIndex] is List<T>;
+    }
+
+    public Type GetVariableType(int columnIndex) {
+      var listType = variableValues[columnIndex].GetType();
+      return listType.GenericTypeArguments.Single();
+    }
+
+    public IList<string> InputVariables { get; private set; }
+    public string TargetVariable { get; private set; } // optional
+    #endregion
+
+    #region Partitions
+    [Storable]
+    public IntRange TrainingPartition { get; set; }
+    [Storable]
+    public IntRange TestPartition { get; set; }
+    #endregion
+
+    #region Transformations
+    [Storable]
+    public IList<ITransformation> Transformations { get; protected set; }
+    #endregion
+
+    #region Validation
+    public bool Validate(string value, out string errorMessage, int columnIndex) {
+      if (columnIndex < 0 || columnIndex > VariableNames.Count()) {
+        throw new ArgumentOutOfRangeException("column index is out of range");
+      }
+
+      bool valid = false;
+      errorMessage = string.Empty;
+      if (VariableHasType<double>(columnIndex)) {
+        if (string.IsNullOrWhiteSpace(value)) {
+          valid = true;
+        } else {
+          double val;
+          valid = double.TryParse(value, out val);
+          if (!valid) {
+            errorMessage = "Invalid Value (Valid Value Format: \"" + FormatPatterns.GetDoubleFormatPattern() + "\")";
+          }
+        }
+      } else if (VariableHasType<string>(columnIndex)) {
+        valid = value != null;
+        if (!valid) {
+          errorMessage = "Invalid Value (string must not be null)";
+        }
+      } else if (VariableHasType<DateTime>(columnIndex)) {
+        DateTime date;
+        valid = DateTime.TryParse(value, out date);
+        if (!valid) {
+          errorMessage = "Invalid Value (Valid Value Format: \"" + CultureInfo.CurrentCulture.DateTimeFormat + "\"";
+        }
+      } else {
+        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
+      }
+
+      return valid;
+    }
+    #endregion
+
+    #region Import & Export
     public void Import(IDataAnalysisProblemData problemData) {
       Dataset dataset = (Dataset)problemData.Dataset;
@@ -107 +397 @@
       TargetVariable = (problemData is IRegressionProblemData) ? ((IRegressionProblemData)problemData).TargetVariable
         : (problemData is IClassificationProblemData) ? ((IClassificationProblemData)problemData).TargetVariable
-        : null;
+          : null;
 
       int columnIndex = 0;
@@ -128 +418 @@
     }
 
-    private void RegisterEventHandler() {
-      Changed += (s, e) => {
-        switch (e.Type) {
-          case DataPreprocessingChangedEventType.DeleteRow:
-            CheckPartitionRanges();
-            break;
-          case DataPreprocessingChangedEventType.Any:
-            CheckPartitionRanges();
-            break;
-          case DataPreprocessingChangedEventType.Transformation:
-            CheckPartitionRanges();
-            break;
-        }
-      };
-    }
-
-    private void CheckPartitionRanges() {
-      int maxRowIndex = Math.Max(0, Rows);
-      TrainingPartition.Start = Math.Min(TrainingPartition.Start, maxRowIndex);
-      TrainingPartition.End = Math.Min(TrainingPartition.End, maxRowIndex);
-      TestPartition.Start = Math.Min(TestPartition.Start, maxRowIndex);
-      TestPartition.End = Math.Min(TestPartition.End, maxRowIndex);
-    }
-
-    protected IList<IList> CopyVariableValues(IList<IList> original) {
-      var copy = new List<IList>(original);
-      for (int i = 0; i < original.Count; ++i) {
-        copy[i] = (IList)Activator.CreateInstance(original[i].GetType(), original[i]);
-      }
-      return copy;
-    }
-
-
-    #region IPreprocessingData Members
-    public abstract T GetCell<T>(int columnIndex, int rowIndex);
-
-    public abstract void SetCell<T>(int columnIndex, int rowIndex, T value);
-
-    public abstract string GetCellAsString(int columnIndex, int rowIndex);
-
-    public abstract string GetVariableName(int columnIndex);
-
-    public abstract int GetColumnIndex(string variableName);
-
-    public abstract bool VariableHasType<T>(int columnIndex);
-
-    [Obsolete("use the index based variant, is faster")]
-    public abstract IList<T> GetValues<T>(string variableName, bool considerSelection);
-
-    public abstract IList<T> GetValues<T>(int columnIndex, bool considerSelection);
-
-    public abstract void SetValues<T>(int columnIndex, IList<T> values);
-
-    public abstract bool SetValue(string value, int columnIndex, int rowIndex);
-
-    public abstract bool Validate(string value, out string errorMessage, int columnIndex);
-
-    public abstract bool AreAllStringColumns(IEnumerable<int> columnIndices);
-
-    public abstract void DeleteRowsWithIndices(IEnumerable<int> rows);
-
-    public abstract void InsertRow(int rowIndex);
-
-    public abstract void DeleteRow(int rowIndex);
-
-    public abstract void InsertColumn<T>(string variableName, int columnIndex);
-
-    public abstract void DeleteColumn(int columnIndex);
-
-    public abstract void RenameColumn(int columnIndex, string name);
-    public abstract void RenameColumns(IList<string> list);
-
-    public abstract Dataset ExportToDataset();
-
-    public abstract void ClearSelection();
-
-    public abstract event EventHandler SelectionChanged;
-    protected abstract void OnSelectionChanged();
+    public Dataset ExportToDataset() {
+      IList<IList> values = new List<IList>();
+
+      for (int i = 0; i < Columns; ++i) {
+        values.Add(variableValues[i]);
+      }
+
+      var dataset = new Dataset(variableNames, values);
+      return dataset;
+    }
+    #endregion
+
+    #region Selection
+    [Storable]
+    protected IDictionary<int, IList<int>> selection;
+    public IDictionary<int, IList<int>> Selection {
+      get { return selection; }
+      set {
+        selection = value;
+        OnSelectionChanged();
+      }
+    }
+    public void ClearSelection() {
+      Selection = new Dictionary<int, IList<int>>();
+    }
+
+    public event EventHandler SelectionChanged;
+    protected void OnSelectionChanged() {
+      var listeners = SelectionChanged;
+      if (listeners != null) listeners(this, EventArgs.Empty);
+    }
+    #endregion
+
+    #region Transactions
+    // Stapshot/History are nost storable/cloneable on purpose
+    private class Snapshot {
+      public IList<IList> VariableValues { get; set; }
+      public IList<string> VariableNames { get; set; }
+
+      public IntRange TrainingPartition { get; set; }
+      public IntRange TestPartition { get; set; }
+      public IList<ITransformation> Transformations { get; set; }
+      public DataPreprocessingChangedEventType ChangedType { get; set; }
+
+      public int ChangedColumn { get; set; }
+      public int ChangedRow { get; set; }
+    }
 
     public event DataPreprocessingChangedEventHandler Changed;
@@ -212 +471 @@
       if (listeners != null) listeners(this, new DataPreprocessingChangedEventArgs(type, column, row));
     }
-    #endregion
+
+    private const int MAX_UNDO_DEPTH = 5;
+
+    private readonly IList<Snapshot> undoHistory = new List<Snapshot>();
+    private readonly Stack<DataPreprocessingChangedEventType> eventStack = new Stack<DataPreprocessingChangedEventType>();
+
+    public bool IsInTransaction { get { return eventStack.Count > 0; } }
+
+    private void SaveSnapshot(DataPreprocessingChangedEventType changedType, int column, int row) {
+      if (IsInTransaction) return;
+
+      var currentSnapshot = new Snapshot {
+        VariableValues = CopyVariableValues(variableValues),
+        VariableNames = new List<string>(variableNames),
+        TrainingPartition = new IntRange(TrainingPartition.Start, TrainingPartition.End),
+        TestPartition = new IntRange(TestPartition.Start, TestPartition.End),
+        Transformations = new List<ITransformation>(Transformations),
+        ChangedType = changedType,
+        ChangedColumn = column,
+        ChangedRow = row
+      };
+
+      if (undoHistory.Count >= MAX_UNDO_DEPTH)
+        undoHistory.RemoveAt(0);
+
+      undoHistory.Add(currentSnapshot);
+    }
+
+    public bool IsUndoAvailable {
+      get { return undoHistory.Count > 0; }
+    }
+
+    public void Undo() {
+      if (IsUndoAvailable) {
+        Snapshot previousSnapshot = undoHistory[undoHistory.Count - 1];
+        variableValues = previousSnapshot.VariableValues;
+        variableNames = previousSnapshot.VariableNames;
+        TrainingPartition = previousSnapshot.TrainingPartition;
+        TestPartition = previousSnapshot.TestPartition;
+        Transformations = previousSnapshot.Transformations;
+        undoHistory.Remove(previousSnapshot);
+        OnChanged(previousSnapshot.ChangedType,
+          previousSnapshot.ChangedColumn,
+          previousSnapshot.ChangedRow);
+      }
+    }
+
+    public void InTransaction(Action action, DataPreprocessingChangedEventType type = DataPreprocessingChangedEventType.Any) {
+      BeginTransaction(type);
+      action();
+      EndTransaction();
+    }
+
+    public void BeginTransaction(DataPreprocessingChangedEventType type) {
+      SaveSnapshot(type, -1, -1);
+      eventStack.Push(type);
+    }
+
+    public void EndTransaction() {
+      if (eventStack.Count == 0)
+        throw new InvalidOperationException("There is no open transaction that can be ended.");
+
+      var @event = eventStack.Pop();
+      OnChanged(@event, -1, -1);
+    }
+    #endregion
+
+    #region Statistics
+    public T GetMin<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Min() : emptyValue;
+    }
+
+    public T GetMax<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Max() : emptyValue;
+    }
+
+    public T GetMean<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      if (typeof(T) == typeof(double)) {
+        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(values.Average()) : emptyValue;
+      }
+      if (typeof(T) == typeof(string)) {
+        return Convert<T>(string.Empty);
+      }
+      if (typeof(T) == typeof(DateTime)) {
+        var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(AggregateAsDouble(values, Enumerable.Average)) : emptyValue;
+      }
+
+      throw new InvalidOperationException(typeof(T) + " not supported");
+    }
+
+    public T GetMedian<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
+        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return doubleValues.Any() ? Convert<T>(doubleValues.Median()) : emptyValue;
+      }
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Quantile(0.5) : emptyValue;
+    }
+
+    public T GetMode<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IEquatable<T> {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.GroupBy(x => x).OrderByDescending(g => g.Count()).Select(g => g.Key).First() : emptyValue;
+    }
+
+    public T GetStandardDeviation<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      if (typeof(T) == typeof(double)) {
+        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(values.StandardDeviation()) : emptyValue;
+      }
+      // For DateTime, std.dev / variance would have to be TimeSpan
+      //if (typeof(T) == typeof(DateTime)) {
+      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.StandardDeviation)) : emptyValue;
+      //}
+      return default(T);
+    }
+
+    public T GetVariance<T>(int columnIndex, bool considerSelection = false, T emptyValue = default(T)) {
+      if (typeof(T) == typeof(double)) {
+        var values = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return values.Any() ? Convert<T>(values.Variance()) : emptyValue;
+      }
+      // DateTime variance often overflows long, thus the corresponding DateTime is invalid
+      //if (typeof(T) == typeof(DateTime)) {
+      //  var values = GetValuesWithoutMissingValues<DateTime>(columnIndex, considerSelection);
+      //  return values.Any() ? Convert<T>(AggregateAsDouble(values, EnumerableStatisticExtensions.Variance)) : emptyValue;
+      //}
+      return default(T);
+    }
+
+    public T GetQuantile<T>(double alpha, int columnIndex, bool considerSelection = false, T emptyValue = default(T)) where T : IComparable<T> {
+      if (typeof(T) == typeof(double)) {// IEnumerable<double> is faster  
+        var doubleValues = GetValuesWithoutMissingValues<double>(columnIndex, considerSelection);
+        return doubleValues.Any() ? Convert<T>(doubleValues.Quantile(alpha)) : emptyValue;
+      }
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.Any() ? values.Quantile(alpha) : emptyValue;
+    }
+
+    public int GetDistinctValues<T>(int columnIndex, bool considerSelection = false) {
+      var values = GetValuesWithoutMissingValues<T>(columnIndex, considerSelection);
+      return values.GroupBy(x => x).Count();
+    }
+
+    private IEnumerable<T> GetValuesWithoutMissingValues<T>(int columnIndex, bool considerSelection) {
+      return GetValues<T>(columnIndex, considerSelection).Where(x => !IsMissingValue(x));
+    }
+
+    private static DateTime AggregateAsDouble(IEnumerable<DateTime> values, Func<IEnumerable<double>, double> func) {
+      return new DateTime((long)(func(values.Select(x => (double)x.Ticks / TimeSpan.TicksPerSecond)) * TimeSpan.TicksPerSecond));
+    }
+    private static T Convert<T>(object obj) { return (T)obj; }
+
+    public int GetMissingValueCount() {
+      int count = 0;
+      for (int i = 0; i < Columns; ++i) {
+        count += GetMissingValueCount(i);
+      }
+      return count;
+    }
+    public int GetMissingValueCount(int columnIndex) {
+      int sum = 0;
+      for (int i = 0; i < Rows; i++) {
+        if (IsCellEmpty(columnIndex, i))
+          sum++;
+      }
+      return sum;
+    }
+    public int GetRowMissingValueCount(int rowIndex) {
+      int sum = 0;
+      for (int i = 0; i < Columns; i++) {
+        if (IsCellEmpty(i, rowIndex))
+          sum++;
+      }
+      return sum;
+    }
+    #endregion
+
+    #region Helpers
+    private static IList<IList> CopyVariableValues(IList<IList> original) {
+      var copy = new List<IList>(original);
+      for (int i = 0; i < original.Count; ++i) {
+        copy[i] = (IList)Activator.CreateInstance(original[i].GetType(), original[i]);
+      }
+      return copy;
+    }
+    #endregion
+  }
+
+  // Adapted from HeuristicLab.Common.EnumerableStatisticExtensions
+  internal static class EnumerableExtensions {
+    public static T Quantile<T>(this IEnumerable<T> values, double alpha) where T : IComparable<T> {
+      T[] valuesArr = values.ToArray();
+      int n = valuesArr.Length;
+      if (n == 0) throw new InvalidOperationException("Enumeration contains no elements.");
+
+      var pos = n * alpha;
+
+      return Select((int)Math.Ceiling(pos) - 1, valuesArr);
+
+    }
+
+    private static T Select<T>(int k, T[] arr) where T : IComparable<T> {
+      int i, ir, j, l, mid, n = arr.Length;
+      T a;
+      l = 0;
+      ir = n - 1;
+      for (;;) {
+        if (ir <= l + 1) {
+          // Active partition contains 1 or 2 elements.
+          if (ir == l + 1 && arr[ir].CompareTo(arr[l]) < 0) {
+            // Case of 2 elements.
+            Swap(arr, l, ir);
+          }
+          return arr[k];
+        } else {
+          mid = (l + ir) >> 1; // Choose median of left, center, and right elements
+          Swap(arr, mid, l + 1); // as partitioning element a. Also
+
+          if (arr[l].CompareTo(arr[ir]) > 0) {  // rearrange so that arr[l] arr[ir] <= arr[l+1],
+            Swap(arr, l, ir); // . arr[ir] >= arr[l+1]
+          }
+
+          if (arr[l + 1].CompareTo(arr[ir]) > 0) {
+            Swap(arr, l + 1, ir);
+          }
+          if (arr[l].CompareTo(arr[l + 1]) > 0) {
+            Swap(arr, l, l + 1);
+          }
+          i = l + 1; // Initialize pointers for partitioning.
+          j = ir;
+          a = arr[l + 1]; // Partitioning element.
+          for (;;) { // Beginning of innermost loop.
+            do i++; while (arr[i].CompareTo(a) < 0); // Scan up to find element > a.
+            do j--; while (arr[j].CompareTo(a) > 0); // Scan down to find element < a.
+            if (j < i) break; // Pointers crossed. Partitioning complete.
+            Swap(arr, i, j);
+          } // End of innermost loop.
+          arr[l + 1] = arr[j]; // Insert partitioning element.
+          arr[j] = a;
+          if (j >= k) ir = j - 1; // Keep active the partition that contains the
+          if (j <= k) l = i; // kth element.
+        }
+      }
+    }
+
+    private static void Swap<T>(T[] arr, int i, int j) {
+      T temp = arr[i];
+      arr[i] = arr[j];
+      arr[j] = temp;
+    }
   }
 }

trunk/sources/HeuristicLab.DataPreprocessing/3.4/HeuristicLab.DataPreprocessing-3.4.csproj

@@ -79 +79 @@
   <ItemGroup>
     <Compile Include="Content\MultiScatterPlotContent.cs" />
+    <Compile Include="Content\PreprocessingContent.cs" />
     <Compile Include="Content\SingleScatterPlotContent.cs" />
     <Compile Include="Content\ScatterPlotContent.cs" />
@@ -90 +91 @@
     <Compile Include="PreprocessingTransformator.cs" />
     <Compile Include="Data\DataPreprocessingChangedEvent.cs" />
-    <Compile Include="Logic\Filter\ComparisonFilter.cs" />
-    <Compile Include="Logic\Filter\IFilter.cs" />
-    <Compile Include="Logic\ManipulationLogic.cs" />
+    <Compile Include="Filter\ComparisonFilter.cs" />
+    <Compile Include="Filter\IFilter.cs" />
     <Compile Include="Data\IPreprocessingData.cs" />
     <Compile Include="Content\FilterContent.cs" />
-    <Compile Include="Logic\FilterLogic.cs" />
     <Compile Include="Content\HistogramContent.cs" />
     <Compile Include="Content\LineChartContent.cs" />
@@ -105 +104 @@
     <Compile Include="Content\DataGridContent.cs" />
     <Compile Include="PreprocessingContext.cs" />
-    <Compile Include="Data\TransactionalPreprocessingData.cs" />
-    <Compile Include="Logic\SearchLogic.cs" />
-    <Compile Include="Logic\StatisticsLogic.cs" />
-    <Compile Include="Data\ITransactionalPreprocessingData.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />

trunk/sources/HeuristicLab.DataPreprocessing/3.4/PreprocessingContext.cs

@@ -71 +71 @@
       if (namedSource != null)
         Name = "Preprocessing " + namedSource.Name;
-      Data = new FilteredPreprocessingData(new TransactionalPreprocessingData(problemData));
+      Data = new FilteredPreprocessingData(new PreprocessingData(problemData));
       OnReset();
       // Reset GUI:

trunk/sources/HeuristicLab.DataPreprocessing/3.4/PreprocessingTransformator.cs

@@ -29 +29 @@
 namespace HeuristicLab.DataPreprocessing {
   public class PreprocessingTransformator {
-    private readonly ITransactionalPreprocessingData preprocessingData;
+    private readonly IPreprocessingData preprocessingData;
 
     private readonly IDictionary<string, IList<double>> originalColumns;
@@ -36 +36 @@
 
     public PreprocessingTransformator(IPreprocessingData preprocessingData) {
-      this.preprocessingData = (ITransactionalPreprocessingData)preprocessingData;
+      this.preprocessingData = preprocessingData;
       originalColumns = new Dictionary<string, IList<double>>();
       renamedColumns = new Dictionary<string, string>();
@@ -68 +68 @@
           preprocessingData.Undo();
         }
-      }
-      catch (Exception e) {
+      } catch (Exception e) {
         preprocessingData.Undo();
         if (string.IsNullOrEmpty(errorMsg)) errorMsg = e.Message;
-      }
-      finally {
+      } finally {
         preprocessingData.EndTransaction();
       }