Changeset 15535 for stable/HeuristicLab.DataPreprocessing/3.4/Content

Timestamp:

12/18/17 16:17:39 (6 years ago)

Author:

pfleck

Message:

#2809: merged r15518,r15534 to stable

Location:

stable

Files:

• . (modified) (1 prop)
• HeuristicLab.DataPreprocessing (modified) (1 prop)
• HeuristicLab.DataPreprocessing/3.4 (modified) (1 prop)
• HeuristicLab.DataPreprocessing/3.4/Content/CorrelationMatrixContent.cs (modified) (3 diffs)
• HeuristicLab.DataPreprocessing/3.4/Content/DataCompletenessChartContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs (modified) (4 diffs)
• HeuristicLab.DataPreprocessing/3.4/Content/FilterContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Content/HistogramContent.cs (modified) (3 diffs)
• HeuristicLab.DataPreprocessing/3.4/Content/LineChartContent.cs (modified) (2 diffs)
• HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Content/MultiScatterPlotContent.cs (modified) (2 diffs)
• HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingChartContent.cs (modified) (4 diffs)
• HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingContent.cs (copied) (copied from trunk/sources/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingContent.cs)
• HeuristicLab.DataPreprocessing/3.4/Content/ScatterPlotContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Content/SingleScatterPlotContent.cs (modified) (2 diffs)
• HeuristicLab.DataPreprocessing/3.4/Content/StatisticsContent.cs (modified) (1 diff)
• HeuristicLab.DataPreprocessing/3.4/Content/TransformationContent.cs (modified) (1 diff)

stable/HeuristicLab.DataPreprocessing/3.4/Content/CorrelationMatrixContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Feature Correlation Matrix", "Represents the feature correlation matrix.")]
-  public class CorrelationMatrixContent : Item, IViewShortcut {
+  [StorableClass]
+  public class CorrelationMatrixContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Gradient; }
     }
 
+    [Storable]
     public PreprocessingContext Context { get; private set; }
-    public ITransactionalPreprocessingData PreprocessingData {
-      get { return Context.Data; }
-    }
+
 
     public DataAnalysisProblemData ProblemData {
@@ -47 +48 @@
     }
 
-    public CorrelationMatrixContent(PreprocessingContext context) {
+    #region Constructor, Cloning & Persistence
+    public CorrelationMatrixContent(PreprocessingContext context)
+      : base(context.Data) {
       Context = context;
     }
@@ -55 +58 @@
       Context = original.Context;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new CorrelationMatrixContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected CorrelationMatrixContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public event DataPreprocessingChangedEventHandler Changed {

stable/HeuristicLab.DataPreprocessing/3.4/Content/DataCompletenessChartContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Data Completeness Chart", "Represents a datacompleteness chart.")]
-
-  public class DataCompletenessChartContent : Item, IViewShortcut {
+  [StorableClass]
+  public class DataCompletenessChartContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.EditBrightnessContrast; }
     }
 
-    public SearchLogic SearchLogic { get; private set; }
-
-    public DataCompletenessChartContent(SearchLogic searchLogic) {
-      SearchLogic = searchLogic;
+    #region Constructor, Cloning & Persistence
+    public DataCompletenessChartContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
     }
 
     public DataCompletenessChartContent(DataCompletenessChartContent content, Cloner cloner)
       : base(content, cloner) {
-      SearchLogic = content.SearchLogic;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new DataCompletenessChartContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected DataCompletenessChartContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

stable/HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs

@@ -27 +27 @@
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Random;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Data Grid", "Represents a data grid.")]
-  public class DataGridContent : Item, IStringConvertibleMatrix, IViewShortcut {
+  [StorableClass]
+  public class DataGridContent : PreprocessingContent, IStringConvertibleMatrix, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Table; }
     }
 
-    public ITransactionalPreprocessingData PreProcessingData { get; private set; }
-
-    public ManipulationLogic ManipulationLogic { get; private set; }
-    public FilterLogic FilterLogic { get; private set; }
-
     public int Rows {
-      get { return PreProcessingData.Rows; }
+      get { return PreprocessingData.Rows; }
       set { }
     }
 
     public int Columns {
-      get { return PreProcessingData.Columns; }
+      get { return PreprocessingData.Columns; }
       set { }
     }
 
     public IEnumerable<string> ColumnNames {
-      get { return PreProcessingData.VariableNames; }
+      get { return PreprocessingData.VariableNames; }
       set { }
     }
@@ -71 +68 @@
 
     public IDictionary<int, IList<int>> Selection {
-      get { return PreProcessingData.Selection; }
-      set { PreProcessingData.Selection = value; }
-    }
-
-    public DataGridContent(ITransactionalPreprocessingData preProcessingData, ManipulationLogic theManipulationLogic, FilterLogic theFilterLogic) {
-      ManipulationLogic = theManipulationLogic;
-      FilterLogic = theFilterLogic;
-      PreProcessingData = preProcessingData;
-    }
-
-    public DataGridContent(DataGridContent dataGridContent, Cloner cloner)
-      : base(dataGridContent, cloner) {
-
+      get { return PreprocessingData.Selection; }
+      set { PreprocessingData.Selection = value; }
+    }
+
+    #region Constructor, Cloning & Persistence
+    public DataGridContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
+    }
+
+    public DataGridContent(DataGridContent original, Cloner cloner)
+      : base(original, cloner) {
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -89 +84 @@
     }
 
+    [StorableConstructor]
+    protected DataGridContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
+
     public void DeleteRows(IEnumerable<int> rows) {
-      PreProcessingData.DeleteRowsWithIndices(rows);
+      PreprocessingData.DeleteRowsWithIndices(rows);
     }
 
     public void DeleteColumn(int column) {
-      PreProcessingData.DeleteColumn(column);
+      PreprocessingData.DeleteColumn(column);
     }
 
     public bool Validate(string value, out string errorMessage, int columnIndex) {
-      return PreProcessingData.Validate(value, out errorMessage, columnIndex);
+      return PreprocessingData.Validate(value, out errorMessage, columnIndex);
     }
 
     public string GetValue(int rowIndex, int columnIndex) {
-      return PreProcessingData.GetCellAsString(columnIndex, rowIndex);
+      return PreprocessingData.GetCellAsString(columnIndex, rowIndex);
     }
 
     public bool SetValue(string value, int rowIndex, int columnIndex) {
-      return PreProcessingData.SetValue(value, columnIndex, rowIndex);
+      return PreprocessingData.SetValue(value, columnIndex, rowIndex);
     }
 
     public event DataPreprocessingChangedEventHandler Changed {
-      add { PreProcessingData.Changed += value; }
-      remove { PreProcessingData.Changed -= value; }
+      add { PreprocessingData.Changed += value; }
+      remove { PreprocessingData.Changed -= value; }
     }
 
@@ -132 +132 @@
 #pragma warning restore 0067
     #endregion
+
+    #region Manipulations
+    private void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, Func<int, double> doubleAggregator = null,
+      Func<int, DateTime> dateTimeAggregator = null, Func<int, string> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var value = doubleAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<double>(column.Key, index, value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var value = dateTimeAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<DateTime>(column.Key, index, value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var value = stringAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<string>(column.Key, index, value);
+          }
+        }
+      });
+    }
+
+    private void ReplaceIndicesByValues(IDictionary<int, IList<int>> cells, Func<int, IEnumerable<double>> doubleAggregator = null,
+      Func<int, IEnumerable<DateTime>> dateTimeAggregator = null, Func<int, IEnumerable<string>> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var values = doubleAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<double>(column.Key, pair.row, pair.value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var values = dateTimeAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<DateTime>(column.Key, pair.row, pair.value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var values = stringAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<string>(column.Key, pair.row, pair.value);
+          }
+        }
+      });
+    }
+
+    public void ReplaceIndicesByMean(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMean<double>(col, considerSelection),
+        col => PreprocessingData.GetMean<DateTime>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMedian<double>(col, considerSelection),
+        col => PreprocessingData.GetMedian<DateTime>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByMode(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMode<double>(col, considerSelection),
+        col => PreprocessingData.GetMode<DateTime>(col, considerSelection),
+        col => PreprocessingData.GetMode<string>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      var rand = new FastRandom();
+      ReplaceIndicesByValues(cells,
+        col => {
+          double min = PreprocessingData.GetMin<double>(col, considerSelection);
+          double max = PreprocessingData.GetMax<double>(col, considerSelection);
+          double range = max - min;
+          return cells[col].Select(_ => rand.NextDouble() * range + min);
+        },
+        col => {
+          var min = PreprocessingData.GetMin<DateTime>(col, considerSelection);
+          var max = PreprocessingData.GetMax<DateTime>(col, considerSelection);
+          double range = (max - min).TotalSeconds;
+          return cells[col].Select(_ => min + TimeSpan.FromSeconds(rand.NextDouble() * range));
+        });
+    }
+
+    public void ReplaceIndicesByString(IDictionary<int, IList<int>> cells, string value) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          foreach (var rowIdx in column.Value) {
+            PreprocessingData.SetValue(value, column.Key, rowIdx);
+          }
+        }
+      });
+    }
+
+
+    public void ReplaceIndicesByLinearInterpolationOfNeighbours(IDictionary<int, IList<int>> cells) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          IList<Tuple<int, int>> startEndings = GetStartAndEndingsForInterpolation(column);
+          foreach (var tuple in startEndings) {
+            Interpolate(column, tuple.Item1, tuple.Item2);
+          }
+        }
+      });
+    }
+
+    private List<Tuple<int, int>> GetStartAndEndingsForInterpolation(KeyValuePair<int, IList<int>> column) {
+      var startEndings = new List<Tuple<int, int>>();
+      var rowIndices = column.Value.OrderBy(x => x).ToList();
+      var count = rowIndices.Count;
+      int start = int.MinValue;
+      for (int i = 0; i < count; ++i) {
+        if (start == int.MinValue) {
+          start = IndexOfPrevPresentValue(column.Key, rowIndices[i]);
+        }
+        if (i + 1 == count || (i + 1 < count && rowIndices[i + 1] - rowIndices[i] > 1)) {
+          int next = IndexOfNextPresentValue(column.Key, rowIndices[i]);
+          if (start > 0 && next < PreprocessingData.Rows) {
+            startEndings.Add(new Tuple<int, int>(start, next));
+          }
+          start = int.MinValue;
+        }
+      }
+      return startEndings;
+    }
+
+    private void Interpolate(KeyValuePair<int, IList<int>> column, int prevIndex, int nextIndex) {
+      int valuesToInterpolate = nextIndex - prevIndex;
+
+      if (PreprocessingData.VariableHasType<double>(column.Key)) {
+        double prev = PreprocessingData.GetCell<double>(column.Key, prevIndex);
+        double next = PreprocessingData.GetCell<double>(column.Key, nextIndex);
+        double interpolationStep = (next - prev) / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          double interpolated = prev + (interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<double>(column.Key, i, interpolated);
+        }
+      } else if (PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+        DateTime prev = PreprocessingData.GetCell<DateTime>(column.Key, prevIndex);
+        DateTime next = PreprocessingData.GetCell<DateTime>(column.Key, nextIndex);
+        double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          DateTime interpolated = prev.AddSeconds(interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<DateTime>(column.Key, i, interpolated);
+        }
+      }
+    }
+
+    private int IndexOfPrevPresentValue(int columnIndex, int start) {
+      int offset = start - 1;
+      while (offset >= 0 && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset--;
+      }
+
+      return offset;
+    }
+
+    private int IndexOfNextPresentValue(int columnIndex, int start) {
+      int offset = start + 1;
+      while (offset < PreprocessingData.Rows && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset++;
+      }
+
+      return offset;
+    }
+
+    public void Shuffle(bool shuffleRangesSeparately) {
+      var random = new FastRandom();
+
+      if (shuffleRangesSeparately) {
+        var ranges = new[] { PreprocessingData.TestPartition, PreprocessingData.TrainingPartition };
+        PreprocessingData.InTransaction(() => {
+          // process all given ranges - e.g. TrainingPartition, TestPartition
+          foreach (IntRange range in ranges) {
+            var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
+            var shuffledIndices = Enumerable.Range(range.Start, range.Size).Shuffle(random).ToArray();
+            for (int i = range.Start, j = 0; i < range.End; i++, j++)
+              indices[i] = shuffledIndices[j];
+
+            ReOrderToIndices(indices);
+          }
+        });
+
+      } else {
+        PreprocessingData.InTransaction(() => {
+          var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
+          indices.ShuffleInPlace(random);
+          ReOrderToIndices(indices);
+        });
+      }
+    }
+
+    public void ReOrderToIndices(int[] indices) {
+      PreprocessingData.InTransaction(() => {
+        for (int i = 0; i < PreprocessingData.Columns; ++i) {
+          if (PreprocessingData.VariableHasType<double>(i))
+            ReOrderToIndices<double>(i, indices);
+          else if (PreprocessingData.VariableHasType<string>(i))
+            ReOrderToIndices<string>(i, indices);
+          else if (PreprocessingData.VariableHasType<DateTime>(i))
+            ReOrderToIndices<DateTime>(i, indices);
+        }
+      });
+    }
+
+    private void ReOrderToIndices<T>(int columnIndex, int[] indices) {
+      var originalData = new List<T>(PreprocessingData.GetValues<T>(columnIndex));
+      if (indices.Length != originalData.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+
+      for (int i = 0; i < indices.Length; i++) {
+        T newValue = originalData[indices[i]];
+        PreprocessingData.SetCell<T>(columnIndex, i, newValue);
+      }
+    }
+    #endregion
   }
 }

stable/HeuristicLab.DataPreprocessing/3.4/Content/FilterContent.cs

@@ -20 +20 @@
 #endregion
 
+using System.Collections.Generic;
 using System.Drawing;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
 using HeuristicLab.DataPreprocessing.Filter;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Filter", "Represents the filter grid.")]
-  public class FilterContent : Item, IViewShortcut {
+  [StorableClass]
+  public class FilterContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Filter; }
     }
-
-    public FilterLogic FilterLogic { get; private set; }
-
+    [Storable]
     public ICheckedItemCollection<IFilter> Filters { get; private set; }
 
+    [Storable]
     public bool IsAndCombination { get; set; }
 
-    public FilterContent(FilterLogic filterLogic) {
+    public IEnumerable<IFilter> ActiveFilters {
+      get { return Filters.Where(f => f.Active && f.ConstraintData != null); }
+    }
+
+    public bool[] GetRemainingRows() {
+      var remainingRows = new bool[PreprocessingData.Rows];
+      if (ActiveFilters.Any()) {
+        var filterResults = ActiveFilters.Select(f => f.Check()).ToList();
+        var rowFilterResults = new bool[filterResults.Count];
+        for (int row = 0; row < remainingRows.Length; row++) {
+          for (int i = 0; i < filterResults.Count; i++)
+            rowFilterResults[i] = filterResults[i][row];
+
+          remainingRows[row] = IsAndCombination
+            ? rowFilterResults.All(x => x)
+            : rowFilterResults.Any(x => x);
+        }
+      } else {
+        // if not filters active => all rows are remaining
+        for (int i = 0; i < remainingRows.Length; i++)
+          remainingRows[i] = true;
+      }
+      return remainingRows;
+    }
+
+    #region Constructor, Cloning & Persistence
+    public FilterContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
       Filters = new CheckedItemCollection<IFilter>();
       IsAndCombination = true;
-      FilterLogic = filterLogic;
     }
 
-    protected FilterContent(FilterContent content, Cloner cloner)
-      : base(content, cloner) {
+    protected FilterContent(FilterContent original, Cloner cloner)
+      : base(original, cloner) {
+      Filters = cloner.Clone(original.Filters);
+      IsAndCombination = original.IsAndCombination;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new FilterContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected FilterContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

stable/HeuristicLab.DataPreprocessing/3.4/Content/HistogramContent.cs

@@ -26 +26 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Histogram", "Represents the histogram grid.")]
+  [StorableClass]
   public class HistogramContent : PreprocessingChartContent {
     public static new Image StaticItemImage {
@@ -34 +36 @@
     }
 
+    [Storable]
     public string GroupingVariableName { get; set; }
 
+    [Storable]
     public int Bins { get; set; }
+    [Storable]
     public bool ExactBins { get; set; }
 
+    [Storable]
     public LegendOrder Order { get; set; }
 
+    #region Constructor, Cloning & Persistence
     public HistogramContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
@@ -47 +54 @@
     }
 
-    public HistogramContent(HistogramContent content, Cloner cloner)
-      : base(content, cloner) {
+    public HistogramContent(HistogramContent original, Cloner cloner)
+      : base(original, cloner) {
+      GroupingVariableName = original.GroupingVariableName;
+      Bins = original.Bins;
+      ExactBins = original.ExactBins;
+      Order = original.Order;
     }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new HistogramContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected HistogramContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public static DataTable CreateHistogram(IFilteredPreprocessingData preprocessingData, string variableName, string groupingVariableName, DataTableVisualProperties.DataTableHistogramAggregation aggregation, LegendOrder legendOrder = LegendOrder.Alphabetically) {

stable/HeuristicLab.DataPreprocessing/3.4/Content/LineChartContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Line Chart", "Represents the line chart grid.")]
+  [StorableClass]
   public class LineChartContent : PreprocessingChartContent {
-    public bool AllInOneMode { get; set; }
-
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Performance; }
     }
 
+    [Storable]
+    public bool AllInOneMode { get; set; }
+
+
+    #region Constructor, Cloning & Persistence
     public LineChartContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
@@ -39 +43 @@
     }
 
-    public LineChartContent(LineChartContent content, Cloner cloner)
-      : base(content, cloner) {
-      this.AllInOneMode = content.AllInOneMode;
+    public LineChartContent(LineChartContent original, Cloner cloner)
+      : base(original, cloner) {
+      AllInOneMode = original.AllInOneMode;
     }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new LineChartContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected LineChartContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

stable/HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs

@@ -20 +20 @@
 #endregion
 
+using System;
+using System.Collections.Generic;
 using System.Drawing;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Manipulation", "Represents the available manipulations on a data set.")]
-  public class ManipulationContent : Item, IViewShortcut {
+  [StorableClass]
+  public class ManipulationContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Method; }
     }
 
-    public ManipulationLogic ManipulationLogic { get; private set; }
-    public SearchLogic SearchLogic { get; private set; }
-    public FilterLogic FilterLogic { get; private set; }
-
-    public ManipulationContent(ManipulationLogic manipulationLogic, SearchLogic searchLogic, FilterLogic filterLogic) {
-      ManipulationLogic = manipulationLogic;
-      SearchLogic = searchLogic;
-      FilterLogic = filterLogic;
+    #region Constructor, Cloning & Persistence
+    public ManipulationContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
     }
 
-    public ManipulationContent(ManipulationContent content, Cloner cloner) : base(content, cloner) { }
-
+    public ManipulationContent(ManipulationContent original, Cloner cloner) :
+      base(original, cloner) {
+    }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new ManipulationContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected ManipulationContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
+
+    public List<int> RowsWithMissingValuesGreater(double percent) {
+      List<int> rows = new List<int>();
+
+      for (int i = 0; i < PreprocessingData.Rows; ++i) {
+        int missingCount = PreprocessingData.GetRowMissingValueCount(i);
+        if (100f / PreprocessingData.Columns * missingCount > percent) {
+          rows.Add(i);
+        }
+      }
+
+      return rows;
+    }
+
+    public List<int> ColumnsWithMissingValuesGreater(double percent) {
+      List<int> columns = new List<int>();
+      for (int i = 0; i < PreprocessingData.Columns; ++i) {
+        int missingCount = PreprocessingData.GetMissingValueCount(i);
+        if (100f / PreprocessingData.Rows * missingCount > percent) {
+          columns.Add(i);
+        }
+      }
+
+      return columns;
+    }
+
+    public List<int> ColumnsWithVarianceSmaller(double variance) {
+      List<int> columns = new List<int>();
+      for (int i = 0; i < PreprocessingData.Columns; ++i) {
+        if (PreprocessingData.VariableHasType<double>(i)) {
+          double columnVariance = PreprocessingData.GetVariance<double>(i);
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        } else if (PreprocessingData.VariableHasType<DateTime>(i)) {
+          double columnVariance = (double)PreprocessingData.GetVariance<DateTime>(i).Ticks / TimeSpan.TicksPerSecond;
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        }
+      }
+      return columns;
+    }
+
+    public void DeleteRowsWithMissingValuesGreater(double percent) {
+      DeleteRows(RowsWithMissingValuesGreater(percent));
+    }
+
+    public void DeleteColumnsWithMissingValuesGreater(double percent) {
+      DeleteColumns(ColumnsWithMissingValuesGreater(percent));
+    }
+
+    public void DeleteColumnsWithVarianceSmaller(double variance) {
+      DeleteColumns(ColumnsWithVarianceSmaller(variance));
+    }
+
+    private void DeleteRows(List<int> rows) {
+      PreprocessingData.InTransaction(() => {
+        foreach (int row in rows.OrderByDescending(x => x)) {
+          PreprocessingData.DeleteRow(row);
+        }
+      });
+    }
+
+    private void DeleteColumns(List<int> columns) {
+      PreprocessingData.InTransaction(() => {
+        foreach (int column in columns.OrderByDescending(x => x)) {
+          PreprocessingData.DeleteColumn(column);
+        }
+      });
+    }
   }
 }

stable/HeuristicLab.DataPreprocessing/3.4/Content/MultiScatterPlotContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Multi Scatter Plot", "Represents a multi scatter plot.")]
+  [StorableClass]
   public class MultiScatterPlotContent : ScatterPlotContent {
     public static new Image StaticItemImage {
@@ -32 +33 @@
     }
 
+    #region Constructor, Cloning & Persistence
     public MultiScatterPlotContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
     }
 
-    public MultiScatterPlotContent(MultiScatterPlotContent content, Cloner cloner)
-      : base(content, cloner) {
+    public MultiScatterPlotContent(MultiScatterPlotContent original, Cloner cloner)
+      : base(original, cloner) {
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new MultiScatterPlotContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected MultiScatterPlotContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

stable/HeuristicLab.DataPreprocessing/3.4/Content/PreprocessingChartContent.cs

@@ -29 +29 @@
 using HeuristicLab.Core;
 using HeuristicLab.Data;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("PreprocessingChart", "Represents a preprocessing chart.")]
-  public class PreprocessingChartContent : Item, IViewShortcut {
+  [StorableClass]
+  public class PreprocessingChartContent : PreprocessingContent, IViewShortcut {
     public enum LegendOrder {
       Alphabetically,
@@ -42 +44 @@
     }
 
-    private ICheckedItemList<StringValue> variableItemList = null;
+    [Storable]
+    private ICheckedItemList<StringValue> variableItemList;
     public ICheckedItemList<StringValue> VariableItemList {
       get {
         if (variableItemList == null)
           variableItemList = CreateVariableItemList(PreprocessingData);
-        return this.variableItemList;
+        return variableItemList;
       }
     }
 
-    public IFilteredPreprocessingData PreprocessingData { get; private set; }
     public event DataPreprocessingChangedEventHandler Changed {
       add { PreprocessingData.Changed += value; }
@@ -57 +59 @@
     }
 
-    public PreprocessingChartContent(IFilteredPreprocessingData preprocessingData) {
-      PreprocessingData = preprocessingData;
+    #region Constructor, Cloning & Persistence
+    public PreprocessingChartContent(IFilteredPreprocessingData preprocessingData)
+       : base(preprocessingData) {
     }
 
-    public PreprocessingChartContent(PreprocessingChartContent content, Cloner cloner)
-      : base(content, cloner) {
-      this.PreprocessingData = content.PreprocessingData;
-      this.variableItemList = cloner.Clone<ICheckedItemList<StringValue>>(variableItemList);
+    public PreprocessingChartContent(PreprocessingChartContent original, Cloner cloner)
+      : base(original, cloner) {
+      variableItemList = cloner.Clone(original.variableItemList);
     }
     public override IDeepCloneable Clone(Cloner cloner) {
       return new PreprocessingChartContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected PreprocessingChartContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public DataRow CreateDataRow(string variableName, DataRowVisualProperties.DataRowChartType chartType) {
@@ -75 +82 @@
 
     public static DataRow CreateDataRow(IFilteredPreprocessingData preprocessingData, string variableName, DataRowVisualProperties.DataRowChartType chartType) {
-      IList<double> values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
-      DataRow row = new DataRow(variableName, "", values);
-      row.VisualProperties.ChartType = chartType;
+      var values = preprocessingData.GetValues<double>(preprocessingData.GetColumnIndex(variableName));
+      var row = new DataRow(variableName, "", values) {
+        VisualProperties = {
+          ChartType = chartType,
+          StartIndexZero = true
+        }
+      };
       return row;
     }

stable/HeuristicLab.DataPreprocessing/3.4/Content/ScatterPlotContent.cs

@@ -25 +25 @@
 using HeuristicLab.Analysis;
 using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Visualization.ChartControlsExtensions;
 
 namespace HeuristicLab.DataPreprocessing {
-
+  [Item("ScatterPlotContent", "")]
+  [StorableClass]
   public abstract class ScatterPlotContent : PreprocessingChartContent {
+    [Storable]
     public string GroupingVariable { get; set; }
 
+    #region Constructor, Cloning & Persistence
     protected ScatterPlotContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
     }
 
-    protected ScatterPlotContent(ScatterPlotContent content, Cloner cloner)
-      : base(content, cloner) {
+    protected ScatterPlotContent(ScatterPlotContent original, Cloner cloner)
+      : base(original, cloner) {
+      GroupingVariable = original.GroupingVariable;
     }
+
+    [StorableConstructor]
+    protected ScatterPlotContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
 
     public static ScatterPlot CreateScatterPlot(IFilteredPreprocessingData preprocessingData, string variableNameX, string variableNameY, string variableNameGroup = "-", LegendOrder legendOrder = LegendOrder.Alphabetically) {

stable/HeuristicLab.DataPreprocessing/3.4/Content/SingleScatterPlotContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
-
   [Item("Scatter Plot", "Represents a scatter plot.")]
+  [StorableClass]
   public class SingleScatterPlotContent : ScatterPlotContent {
     public static new Image StaticItemImage {
@@ -32 +33 @@
     }
 
+    [Storable]
     public string SelectedXVariable { get; set; }
+    [Storable]
     public string SelectedYVariable { get; set; }
 
+    #region Constructor, Cloning & Persistence
     public SingleScatterPlotContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
     }
 
-    public SingleScatterPlotContent(SingleScatterPlotContent content, Cloner cloner)
-      : base(content, cloner) {
-      this.SelectedXVariable = content.SelectedXVariable;
-      this.SelectedYVariable = content.SelectedYVariable;
-      this.GroupingVariable = content.GroupingVariable;
+    public SingleScatterPlotContent(SingleScatterPlotContent original, Cloner cloner)
+      : base(original, cloner) {
+      SelectedXVariable = original.SelectedXVariable;
+      SelectedYVariable = original.SelectedYVariable;
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new SingleScatterPlotContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected SingleScatterPlotContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }

stable/HeuristicLab.DataPreprocessing/3.4/Content/StatisticsContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Statistics", "Represents the statistics grid.")]
-  public class StatisticsContent : Item, IViewShortcut {
+  [StorableClass]
+  public class StatisticsContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Object; }
     }
 
-    public ITransactionalPreprocessingData PreprocessingData { get; private set; }
-    public StatisticsLogic StatisticsLogic { get; private set; }
-
-    public StatisticsContent(ITransactionalPreprocessingData preProcessingData, StatisticsLogic statisticsLogic) {
-      PreprocessingData = preProcessingData;
-      StatisticsLogic = statisticsLogic;
+    #region Constructor, Cloning & Persistence
+    public StatisticsContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
     }
 
-    public StatisticsContent(StatisticsContent content, Cloner cloner)
-      : base(content, cloner) {
+    public StatisticsContent(StatisticsContent original, Cloner cloner)
+      : base(original, cloner) {
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new StatisticsContent(this, cloner);
     }
 
+    [StorableConstructor]
+    protected StatisticsContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
+
     public event DataPreprocessingChangedEventHandler Changed {
-      add { StatisticsLogic.Changed += value; }
-      remove { StatisticsLogic.Changed -= value; }
+      add { PreprocessingData.Changed += value; }
+      remove { PreprocessingData.Changed -= value; }
     }
   }

stable/HeuristicLab.DataPreprocessing/3.4/Content/TransformationContent.cs

@@ -23 +23 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
 
 namespace HeuristicLab.DataPreprocessing {
   [Item("Transformation", "Represents the transformation grid.")]
-  public class TransformationContent : Item, IViewShortcut {
+  [StorableClass]
+  public class TransformationContent : PreprocessingContent, IViewShortcut {
     public static new Image StaticItemImage {
       get { return HeuristicLab.Common.Resources.VSImageLibrary.Method; }
     }
 
-    public IPreprocessingData Data { get; private set; }
-    public FilterLogic FilterLogic { get; private set; }
-
+    [Storable]
     public ICheckedItemList<ITransformation> CheckedTransformationList { get; private set; }
 
-    public TransformationContent(IPreprocessingData data, FilterLogic filterLogic) {
-      Data = data;
+    #region Constructor, Cloning & Persistence
+    public TransformationContent(IFilteredPreprocessingData preprocessingData)
+      : base(preprocessingData) {
       CheckedTransformationList = new CheckedItemList<ITransformation>();
-      FilterLogic = filterLogic;
     }
 
     public TransformationContent(TransformationContent original, Cloner cloner)
       : base(original, cloner) {
-      Data = original.Data;
-      CheckedTransformationList = new CheckedItemList<ITransformation>(original.CheckedTransformationList);
+      CheckedTransformationList = cloner.Clone(original.CheckedTransformationList);
     }
-
     public override IDeepCloneable Clone(Cloner cloner) {
       return new TransformationContent(this, cloner);
     }
+
+    [StorableConstructor]
+    protected TransformationContent(bool deserializing)
+      : base(deserializing) { }
+    #endregion
   }
 }