Changeset 15285 for branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4

Timestamp:

07/26/17 14:12:27 (7 years ago)

Author:

pfleck

Message:

#2809 removed ManipulationLogic

Location:

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4

Files:

• Content/DataGridContent.cs (modified) (4 diffs)
• Content/ManipulationContent.cs (modified) (3 diffs)
• Data/PreprocessingData.cs (modified) (2 diffs)
• Filter (copied) (copied from branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Logic/Filter)
• HeuristicLab.DataPreprocessing-3.4.csproj (modified) (2 diffs)
• Logic (deleted)

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/DataGridContent.cs

@@ -28 +28 @@
 using HeuristicLab.Data;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Random;
 
 namespace HeuristicLab.DataPreprocessing {
@@ -37 +38 @@
     }
 
-    [Storable]
-    public ManipulationLogic ManipulationLogic { get; private set; }
-
     public int Rows {
       get { return PreprocessingData.Rows; }
@@ -75 +73 @@
 
     #region Constructor, Cloning & Persistence
-    public DataGridContent(IFilteredPreprocessingData preprocessingData, ManipulationLogic manipulationLogic)
+    public DataGridContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
-      ManipulationLogic = manipulationLogic;
     }
 
     public DataGridContent(DataGridContent original, Cloner cloner)
       : base(original, cloner) {
-      ManipulationLogic = cloner.Clone(original.ManipulationLogic);
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -136 +132 @@
 #pragma warning restore 0067
     #endregion
+
+    #region Manipulations
+    private void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, Func<int, double> doubleAggregator = null,
+      Func<int, DateTime> dateTimeAggregator = null, Func<int, string> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var value = doubleAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<double>(column.Key, index, value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var value = dateTimeAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<DateTime>(column.Key, index, value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var value = stringAggregator(column.Key);
+            foreach (int index in column.Value)
+              PreprocessingData.SetCell<string>(column.Key, index, value);
+          }
+        }
+      });
+    }
+
+    private void ReplaceIndicesByValues(IDictionary<int, IList<int>> cells, Func<int, IEnumerable<double>> doubleAggregator = null,
+      Func<int, IEnumerable<DateTime>> dateTimeAggregator = null, Func<int, IEnumerable<string>> stringAggregator = null) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          if (doubleAggregator != null && PreprocessingData.VariableHasType<double>(column.Key)) {
+            var values = doubleAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<double>(column.Key, pair.row, pair.value);
+          } else if (dateTimeAggregator != null && PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+            var values = dateTimeAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<DateTime>(column.Key, pair.row, pair.value);
+          } else if (stringAggregator != null && PreprocessingData.VariableHasType<string>(column.Key)) {
+            var values = stringAggregator(column.Key);
+            foreach (var pair in column.Value.Zip(values, (row, value) => new { row, value }))
+              PreprocessingData.SetCell<string>(column.Key, pair.row, pair.value);
+          }
+        }
+      });
+    }
+
+    public void ReplaceIndicesByMean(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMean<double>(col, considerSelection),
+        col => PreprocessingData.GetMean<DateTime>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMedian<double>(col, considerSelection),
+        col => PreprocessingData.GetMedian<DateTime>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByMode(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      ReplaceIndicesByValue(cells,
+        col => PreprocessingData.GetMode<double>(col, considerSelection),
+        col => PreprocessingData.GetMode<DateTime>(col, considerSelection),
+        col => PreprocessingData.GetMode<string>(col, considerSelection));
+    }
+
+    public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
+      var rand = new FastRandom();
+      ReplaceIndicesByValues(cells,
+        col => {
+          double min = PreprocessingData.GetMin<double>(col, considerSelection);
+          double max = PreprocessingData.GetMax<double>(col, considerSelection);
+          double range = max - min;
+          return cells[col].Select(_ => rand.NextDouble() * range + min);
+        },
+        col => {
+          var min = PreprocessingData.GetMin<DateTime>(col, considerSelection);
+          var max = PreprocessingData.GetMax<DateTime>(col, considerSelection);
+          double range = (max - min).TotalSeconds;
+          return cells[col].Select(_ => min + TimeSpan.FromSeconds(rand.NextDouble() * range));
+        });
+    }
+
+    public void ReplaceIndicesByString(IDictionary<int, IList<int>> cells, string value) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          foreach (var rowIdx in column.Value) {
+            PreprocessingData.SetValue(value, column.Key, rowIdx);
+          }
+        }
+      });
+    }
+
+
+    public void ReplaceIndicesByLinearInterpolationOfNeighbours(IDictionary<int, IList<int>> cells) {
+      PreprocessingData.InTransaction(() => {
+        foreach (var column in cells) {
+          IList<Tuple<int, int>> startEndings = GetStartAndEndingsForInterpolation(column);
+          foreach (var tuple in startEndings) {
+            Interpolate(column, tuple.Item1, tuple.Item2);
+          }
+        }
+      });
+    }
+
+    private List<Tuple<int, int>> GetStartAndEndingsForInterpolation(KeyValuePair<int, IList<int>> column) {
+      var startEndings = new List<Tuple<int, int>>();
+      var rowIndices = column.Value.OrderBy(x => x).ToList();
+      var count = rowIndices.Count;
+      int start = int.MinValue;
+      for (int i = 0; i < count; ++i) {
+        if (start == int.MinValue) {
+          start = IndexOfPrevPresentValue(column.Key, rowIndices[i]);
+        }
+        if (i + 1 == count || (i + 1 < count && rowIndices[i + 1] - rowIndices[i] > 1)) {
+          int next = IndexOfNextPresentValue(column.Key, rowIndices[i]);
+          if (start > 0 && next < PreprocessingData.Rows) {
+            startEndings.Add(new Tuple<int, int>(start, next));
+          }
+          start = int.MinValue;
+        }
+      }
+      return startEndings;
+    }
+
+    private void Interpolate(KeyValuePair<int, IList<int>> column, int prevIndex, int nextIndex) {
+      int valuesToInterpolate = nextIndex - prevIndex;
+
+      if (PreprocessingData.VariableHasType<double>(column.Key)) {
+        double prev = PreprocessingData.GetCell<double>(column.Key, prevIndex);
+        double next = PreprocessingData.GetCell<double>(column.Key, nextIndex);
+        double interpolationStep = (next - prev) / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          double interpolated = prev + (interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<double>(column.Key, i, interpolated);
+        }
+      } else if (PreprocessingData.VariableHasType<DateTime>(column.Key)) {
+        DateTime prev = PreprocessingData.GetCell<DateTime>(column.Key, prevIndex);
+        DateTime next = PreprocessingData.GetCell<DateTime>(column.Key, nextIndex);
+        double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
+
+        for (int i = prevIndex; i < nextIndex; ++i) {
+          DateTime interpolated = prev.AddSeconds(interpolationStep * (i - prevIndex));
+          PreprocessingData.SetCell<DateTime>(column.Key, i, interpolated);
+        }
+      }
+    }
+
+    private int IndexOfPrevPresentValue(int columnIndex, int start) {
+      int offset = start - 1;
+      while (offset >= 0 && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset--;
+      }
+
+      return offset;
+    }
+
+    private int IndexOfNextPresentValue(int columnIndex, int start) {
+      int offset = start + 1;
+      while (offset < PreprocessingData.Rows && PreprocessingData.IsCellEmpty(columnIndex, offset)) {
+        offset++;
+      }
+
+      return offset;
+    }
+
+    public void Shuffle(bool shuffleRangesSeparately) {
+      var random = new FastRandom();
+
+      if (shuffleRangesSeparately) {
+        var ranges = new[] { PreprocessingData.TestPartition, PreprocessingData.TrainingPartition };
+        PreprocessingData.InTransaction(() => {
+          // process all given ranges - e.g. TrainingPartition, TestPartition
+          foreach (IntRange range in ranges) {
+            var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
+            var shuffledIndices = Enumerable.Range(range.Start, range.Size).Shuffle(random).ToArray();
+            for (int i = range.Start, j = 0; i < range.End; i++, j++)
+              indices[i] = shuffledIndices[j];
+
+            ReOrderToIndices(indices);
+          }
+        });
+
+      } else {
+        PreprocessingData.InTransaction(() => {
+          var indices = Enumerable.Range(0, PreprocessingData.Rows).ToArray();
+          indices.ShuffleInPlace(random);
+          ReOrderToIndices(indices);
+        });
+      }
+    }
+
+    public void ReOrderToIndices(int[] indices) {
+      PreprocessingData.InTransaction(() => {
+        for (int i = 0; i < PreprocessingData.Columns; ++i) {
+          if (PreprocessingData.VariableHasType<double>(i))
+            ReOrderToIndices<double>(i, indices);
+          else if (PreprocessingData.VariableHasType<string>(i))
+            ReOrderToIndices<string>(i, indices);
+          else if (PreprocessingData.VariableHasType<DateTime>(i))
+            ReOrderToIndices<DateTime>(i, indices);
+        }
+      });
+    }
+
+    private void ReOrderToIndices<T>(int columnIndex, int[] indices) {
+      var originalData = new List<T>(PreprocessingData.GetValues<T>(columnIndex));
+      if (indices.Length != originalData.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
+
+      for (int i = 0; i < indices.Length; i++) {
+        T newValue = originalData[indices[i]];
+        PreprocessingData.SetCell<T>(columnIndex, i, newValue);
+      }
+    }
+    #endregion
   }
 }

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Content/ManipulationContent.cs

@@ -20 +20 @@
 #endregion
 
+using System;
+using System.Collections.Generic;
 using System.Drawing;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -33 +36 @@
     }
 
-    [Storable]
-    public ManipulationLogic ManipulationLogic { get; private set; }
-
     #region Constructor, Cloning & Persistence
-    public ManipulationContent(IFilteredPreprocessingData preprocessingData, ManipulationLogic manipulationLogic)
+    public ManipulationContent(IFilteredPreprocessingData preprocessingData)
       : base(preprocessingData) {
-      ManipulationLogic = manipulationLogic;
     }
 
     public ManipulationContent(ManipulationContent original, Cloner cloner) :
       base(original, cloner) {
-      ManipulationLogic = cloner.Clone(original.ManipulationLogic);
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -54 +52 @@
       : base(deserializing) { }
     #endregion
+
+    public List<int> RowsWithMissingValuesGreater(double percent) {
+      List<int> rows = new List<int>();
+
+      for (int i = 0; i < PreprocessingData.Rows; ++i) {
+        int missingCount = PreprocessingData.GetRowMissingValueCount(i);
+        if (100f / PreprocessingData.Columns * missingCount > percent) {
+          rows.Add(i);
+        }
+      }
+
+      return rows;
+    }
+
+    public List<int> ColumnsWithMissingValuesGreater(double percent) {
+      List<int> columns = new List<int>();
+      for (int i = 0; i < PreprocessingData.Columns; ++i) {
+        int missingCount = PreprocessingData.GetMissingValueCount(i);
+        if (100f / PreprocessingData.Rows * missingCount > percent) {
+          columns.Add(i);
+        }
+      }
+
+      return columns;
+    }
+
+    public List<int> ColumnsWithVarianceSmaller(double variance) {
+      List<int> columns = new List<int>();
+      for (int i = 0; i < PreprocessingData.Columns; ++i) {
+        if (PreprocessingData.VariableHasType<double>(i)) {
+          double columnVariance = PreprocessingData.GetVariance<double>(i);
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        } else if (PreprocessingData.VariableHasType<DateTime>(i)) {
+          double columnVariance = (double)PreprocessingData.GetVariance<DateTime>(i).Ticks / TimeSpan.TicksPerSecond;
+          if (columnVariance < variance) {
+            columns.Add(i);
+          }
+        }
+      }
+      return columns;
+    }
+
+    public void DeleteRowsWithMissingValuesGreater(double percent) {
+      DeleteRows(RowsWithMissingValuesGreater(percent));
+    }
+
+    public void DeleteColumnsWithMissingValuesGreater(double percent) {
+      DeleteColumns(ColumnsWithMissingValuesGreater(percent));
+    }
+
+    public void DeleteColumnsWithVarianceSmaller(double variance) {
+      DeleteColumns(ColumnsWithVarianceSmaller(variance));
+    }
+
+    private void DeleteRows(List<int> rows) {
+      PreprocessingData.InTransaction(() => {
+        foreach (int row in rows.OrderByDescending(x => x)) {
+          PreprocessingData.DeleteRow(row);
+        }
+      });
+    }
+
+    private void DeleteColumns(List<int> columns) {
+      PreprocessingData.InTransaction(() => {
+        foreach (int column in columns.OrderByDescending(x => x)) {
+          PreprocessingData.DeleteColumn(column);
+        }
+      });
+    }
   }
 }

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/Data/PreprocessingData.cs

@@ -151 +151 @@
       if (VariableHasType<double>(columnIndex)) {
         double val;
-        valid = double.TryParse(value, out val);
+        if (string.IsNullOrWhiteSpace(value)) {
+          val = double.NaN;
+          valid = true;
+        } else {
+          valid = double.TryParse(value, out val);
+        }
         if (valid)
           SetCell(columnIndex, rowIndex, val);
@@ -357 +362 @@
       errorMessage = string.Empty;
       if (VariableHasType<double>(columnIndex)) {
-        double val;
-        valid = double.TryParse(value, out val);
-        if (!valid) {
-          errorMessage = "Invalid Value (Valid Value Format: \"" + FormatPatterns.GetDoubleFormatPattern() + "\")";
+        if (string.IsNullOrWhiteSpace(value)) {
+          valid = true;
+        } else {
+          double val;
+          valid = double.TryParse(value, out val);
+          if (!valid) {
+            errorMessage = "Invalid Value (Valid Value Format: \"" + FormatPatterns.GetDoubleFormatPattern() + "\")";
+          }
         }
       } else if (VariableHasType<string>(columnIndex)) {

branches/DataPreprocessing Cleanup/HeuristicLab.DataPreprocessing/3.4/HeuristicLab.DataPreprocessing-3.4.csproj

@@ -131 +131 @@
     <Compile Include="PreprocessingTransformator.cs" />
     <Compile Include="Data\DataPreprocessingChangedEvent.cs" />
-    <Compile Include="Logic\Filter\ComparisonFilter.cs" />
-    <Compile Include="Logic\Filter\IFilter.cs" />
-    <Compile Include="Logic\ManipulationLogic.cs" />
+    <Compile Include="Filter\ComparisonFilter.cs" />
+    <Compile Include="Filter\IFilter.cs" />
     <Compile Include="Data\IPreprocessingData.cs" />
     <Compile Include="Content\FilterContent.cs" />
@@ -152 +151 @@
     <None Include="HeuristicLab.snk" />
   </ItemGroup>
+  <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <PropertyGroup>