Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.DataPreprocessing/3.4/Logic/ManipulationLogic.cs @ 15344

Last change on this file since 15344 was 15110, checked in by pfleck, 7 years ago

#2709: merged branch to trunk

File size: 14.4 KB
RevLine 
[10539]1#region License Information
2/* HeuristicLab
[14185]3 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[10539]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
[10193]23using System.Collections.Generic;
24using System.Linq;
[10249]25using HeuristicLab.Data;
[14886]26using HeuristicLab.Random;
[10193]27
[10249]28namespace HeuristicLab.DataPreprocessing {
[13508]29  public class ManipulationLogic {
[11070]30    private readonly ITransactionalPreprocessingData preprocessingData;
[13508]31    private readonly StatisticsLogic statisticsLogic;
32    private readonly SearchLogic searchLogic;
[10193]33
[11002]34    public IEnumerable<string> VariableNames {
35      get { return preprocessingData.VariableNames; }
36    }
37
38    public ITransactionalPreprocessingData PreProcessingData {
39      get { return preprocessingData; }
40    }
41
[15110]42    public ManipulationLogic(ITransactionalPreprocessingData preprocessingData, SearchLogic theSearchLogic, StatisticsLogic theStatisticsLogic) {
43      this.preprocessingData = preprocessingData;
[10249]44      searchLogic = theSearchLogic;
[10615]45      statisticsLogic = theStatisticsLogic;
[10249]46    }
[10193]47
[10367]48    public void ReplaceIndicesByValue<T>(int columnIndex, IEnumerable<int> rowIndices, T value) {
49      foreach (int index in rowIndices) {
50        preprocessingData.SetCell<T>(columnIndex, index, value);
[10249]51      }
52    }
[10193]53
[13508]54    public void ReplaceIndicesByAverageValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
[10612]55      preprocessingData.InTransaction(() => {
56        foreach (var column in cells) {
[11156]57          if (preprocessingData.VariableHasType<double>(column.Key)) {
[10809]58            double average = statisticsLogic.GetAverage(column.Key, considerSelection);
[10615]59            ReplaceIndicesByValue<double>(column.Key, column.Value, average);
[11156]60          } else if (preprocessingData.VariableHasType<DateTime>(column.Key)) {
[10809]61            DateTime average = statisticsLogic.GetAverageDateTime(column.Key, considerSelection);
[10615]62            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, average);
63          }
[10612]64        }
65      });
[10249]66    }
[10193]67
[13508]68    public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
[10612]69      preprocessingData.InTransaction(() => {
70        foreach (var column in cells) {
[11156]71          if (preprocessingData.VariableHasType<double>(column.Key)) {
[10809]72            double median = statisticsLogic.GetMedian(column.Key, considerSelection);
[10615]73            ReplaceIndicesByValue<double>(column.Key, column.Value, median);
[11156]74          } else if (preprocessingData.VariableHasType<DateTime>(column.Key)) {
[10809]75            DateTime median = statisticsLogic.GetMedianDateTime(column.Key, considerSelection);
[10615]76            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, median);
77          }
[10612]78        }
79      });
[10249]80    }
[10193]81
[13508]82    public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
[10612]83      preprocessingData.InTransaction(() => {
[14886]84        System.Random r = new System.Random();
[10193]85
[10612]86        foreach (var column in cells) {
[11156]87          if (preprocessingData.VariableHasType<double>(column.Key)) {
[13935]88            double max = statisticsLogic.GetMax<double>(column.Key, double.NaN, considerSelection);
89            double min = statisticsLogic.GetMin<double>(column.Key, double.NaN, considerSelection);
[10615]90            double randMultiplier = (max - min);
91            foreach (int index in column.Value) {
92              double rand = r.NextDouble() * randMultiplier + min;
93              preprocessingData.SetCell<double>(column.Key, index, rand);
94            }
[11156]95          } else if (preprocessingData.VariableHasType<DateTime>(column.Key)) {
[13935]96            DateTime min = statisticsLogic.GetMin<DateTime>(column.Key, DateTime.MinValue, considerSelection);
97            DateTime max = statisticsLogic.GetMax<DateTime>(column.Key, DateTime.MinValue, considerSelection);
[10615]98            double randMultiplier = (max - min).TotalSeconds;
99            foreach (int index in column.Value) {
100              double rand = r.NextDouble() * randMultiplier;
101              preprocessingData.SetCell<DateTime>(column.Key, index, min.AddSeconds(rand));
102            }
[10612]103          }
[10590]104        }
[10612]105      });
[10249]106    }
[10193]107
[10672]108    public void ReplaceIndicesByLinearInterpolationOfNeighbours(IDictionary<int, IList<int>> cells) {
[10612]109      preprocessingData.InTransaction(() => {
110        foreach (var column in cells) {
[10820]111          IList<Tuple<int, int>> startEndings = GetStartAndEndingsForInterpolation(column);
112          foreach (var tuple in startEndings) {
113            Interpolate(column, tuple.Item1, tuple.Item2);
114          }
115        }
116      });
117    }
118
119    private List<Tuple<int, int>> GetStartAndEndingsForInterpolation(KeyValuePair<int, IList<int>> column) {
120      List<Tuple<int, int>> startEndings = new List<Tuple<int, int>>();
121      var rowIndices = column.Value;
122      rowIndices = rowIndices.OrderBy(x => x).ToList();
123      var count = rowIndices.Count;
124      int start = int.MinValue;
125      for (int i = 0; i < count; ++i) {
126        if (start == int.MinValue) {
127          start = indexOfPrevPresentValue(column.Key, rowIndices[i]);
128        }
129        if (i + 1 == count || (i + 1 < count && rowIndices[i + 1] - rowIndices[i] > 1)) {
130          int next = indexOfNextPresentValue(column.Key, rowIndices[i]);
131          if (start > 0 && next < preprocessingData.Rows) {
132            startEndings.Add(new Tuple<int, int>(start, next));
133          }
134          start = int.MinValue;
135        }
136      }
137      return startEndings;
138    }
139
140    public void ReplaceIndicesBySmoothing(IDictionary<int, IList<int>> cells) {
141      preprocessingData.InTransaction(() => {
142        foreach (var column in cells) {
143          int countValues = preprocessingData.Rows;
144
[10621]145          foreach (int index in column.Value) {
146            // dont replace first or last values
147            if (index > 0 && index < countValues) {
148              int prevIndex = indexOfPrevPresentValue(column.Key, index);
149              int nextIndex = indexOfNextPresentValue(column.Key, index);
150
151              // no neighbours found
[10820]152              if (prevIndex < 0 || nextIndex >= countValues) {
[10621]153                continue;
154              }
155
[10820]156              Interpolate(column, prevIndex, nextIndex);
[10590]157            }
[10249]158          }
[10193]159        }
[10612]160      });
[10249]161    }
[10193]162
[10820]163    private void Interpolate(KeyValuePair<int, IList<int>> column, int prevIndex, int nextIndex) {
164      int valuesToInterpolate = nextIndex - prevIndex;
165
[11156]166      if (preprocessingData.VariableHasType<double>(column.Key)) {
[10820]167        double prev = preprocessingData.GetCell<double>(column.Key, prevIndex);
168        double next = preprocessingData.GetCell<double>(column.Key, nextIndex);
169        double interpolationStep = (next - prev) / valuesToInterpolate;
170
171        for (int i = prevIndex; i < nextIndex; ++i) {
172          double interpolated = prev + (interpolationStep * (i - prevIndex));
173          preprocessingData.SetCell<double>(column.Key, i, interpolated);
174        }
[11156]175      } else if (preprocessingData.VariableHasType<DateTime>(column.Key)) {
[10820]176        DateTime prev = preprocessingData.GetCell<DateTime>(column.Key, prevIndex);
177        DateTime next = preprocessingData.GetCell<DateTime>(column.Key, nextIndex);
178        double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
179
180        for (int i = prevIndex; i < nextIndex; ++i) {
181          DateTime interpolated = prev.AddSeconds(interpolationStep * (i - prevIndex));
182          preprocessingData.SetCell<DateTime>(column.Key, i, interpolated);
183        }
184      }
185    }
186
[10367]187    private int indexOfPrevPresentValue(int columnIndex, int start) {
[10249]188      int offset = start - 1;
[10367]189      while (offset >= 0 && searchLogic.IsMissingValue(columnIndex, offset)) {
[10249]190        offset--;
191      }
[10234]192
[10249]193      return offset;
194    }
[10234]195
[10367]196    private int indexOfNextPresentValue(int columnIndex, int start) {
[10249]197      int offset = start + 1;
[10367]198      while (offset < preprocessingData.Rows && searchLogic.IsMissingValue(columnIndex, offset)) {
[10249]199        offset++;
200      }
[10234]201
[10249]202      return offset;
203    }
[10234]204
[13508]205    public void ReplaceIndicesByMostCommonValue(IDictionary<int, IList<int>> cells, bool considerSelection = false) {
[10612]206      preprocessingData.InTransaction(() => {
207        foreach (var column in cells) {
[11156]208          if (preprocessingData.VariableHasType<double>(column.Key)) {
[13935]209            ReplaceIndicesByValue<double>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<double>(column.Key, double.NaN, considerSelection));
[11156]210          } else if (preprocessingData.VariableHasType<string>(column.Key)) {
[13935]211            ReplaceIndicesByValue<string>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<string>(column.Key, string.Empty, considerSelection));
[11156]212          } else if (preprocessingData.VariableHasType<DateTime>(column.Key)) {
[13935]213            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<DateTime>(column.Key, DateTime.MinValue, considerSelection));
[10612]214          } else {
215            throw new ArgumentException("column with index: " + column.Key + " contains a non supported type.");
216          }
[10590]217        }
[10612]218      });
[10249]219    }
[10218]220
[11403]221    public void Shuffle(bool shuffleRangesSeparately) {
[14886]222      var random = new FastRandom();
223
[11380]224      if (shuffleRangesSeparately) {
[14886]225        var ranges = new[] { preprocessingData.TestPartition, preprocessingData.TrainingPartition };
[11380]226        preprocessingData.InTransaction(() => {
227          // process all given ranges - e.g. TrainingPartition, TestPartition
228          foreach (IntRange range in ranges) {
[14886]229            var indices = Enumerable.Range(0, preprocessingData.Rows).ToArray();
230            var shuffledIndices = Enumerable.Range(range.Start, range.Size).Shuffle(random).ToArray();
231            for (int i = range.Start, j = 0; i < range.End; i++, j++)
232              indices[i] = shuffledIndices[j];
[10218]233
[14886]234            ReOrderToIndices(indices);
[10612]235          }
[11380]236        });
[14886]237
[11380]238      } else {
239        preprocessingData.InTransaction(() => {
[14886]240          var indices = Enumerable.Range(0, preprocessingData.Rows);
241          var shuffledIndices = indices.Shuffle(random).ToArray();
242          ReOrderToIndices(shuffledIndices);
[11380]243        });
244      }
[10253]245    }
246
[14886]247    public void ReOrderToIndices(int[] indices) {
[10612]248      preprocessingData.InTransaction(() => {
249        for (int i = 0; i < preprocessingData.Columns; ++i) {
[11156]250          if (preprocessingData.VariableHasType<double>(i)) {
[14886]251            ReOrderToIndices<double>(i, indices);
[11156]252          } else if (preprocessingData.VariableHasType<string>(i)) {
[14886]253            ReOrderToIndices<string>(i, indices);
[11156]254          } else if (preprocessingData.VariableHasType<DateTime>(i)) {
[14886]255            ReOrderToIndices<DateTime>(i, indices);
[10612]256          }
[10249]257        }
[10612]258      });
[10249]259    }
[10218]260
[14886]261    private void ReOrderToIndices<T>(int columnIndex, int[] indices) {
[10811]262      List<T> originalData = new List<T>(preprocessingData.GetValues<T>(columnIndex));
[14886]263      if (indices.Length != originalData.Count) throw new InvalidOperationException("The number of provided indices does not match the values.");
[10308]264
[14886]265      for (int i = 0; i < indices.Length; i++) {
266        int originalIndex = i;
267        int replaceIndex = indices[i];
[10218]268
[10308]269        T replaceValue = originalData.ElementAt<T>(replaceIndex);
[10367]270        preprocessingData.SetCell<T>(columnIndex, originalIndex, replaceValue);
[10249]271      }
[10193]272    }
[10672]273
274    public void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, string value) {
275      preprocessingData.InTransaction(() => {
276        foreach (var column in cells) {
277          foreach (var rowIdx in column.Value) {
[11002]278            preprocessingData.SetValue(value, column.Key, rowIdx);
[10672]279          }
280        }
281      });
282    }
[10711]283
284
[10715]285    public List<int> RowsWithMissingValuesGreater(double percent) {
[10820]286      List<int> rows = new List<int>();
[10715]287
[10820]288      for (int i = 0; i < preprocessingData.Rows; ++i) {
[10711]289        int missingCount = statisticsLogic.GetRowMissingValueCount(i);
[10820]290        if (100f / preprocessingData.Columns * missingCount > percent) {
[10715]291          rows.Add(i);
[10711]292        }
293      }
[10715]294
295      return rows;
[10711]296    }
297
[10715]298    public List<int> ColumnsWithMissingValuesGreater(double percent) {
299      List<int> columns = new List<int>();
[10737]300      for (int i = 0; i < preprocessingData.Columns; ++i) {
[10711]301        int missingCount = statisticsLogic.GetMissingValueCount(i);
[10737]302        if (100f / preprocessingData.Rows * missingCount > percent) {
[10715]303          columns.Add(i);
[10711]304        }
305      }
[10715]306
307      return columns;
[10711]308    }
309
[10715]310    public List<int> ColumnsWithVarianceSmaller(double variance) {
311      List<int> columns = new List<int>();
[10737]312      for (int i = 0; i < preprocessingData.Columns; ++i) {
[11156]313        if (preprocessingData.VariableHasType<double>(i) || preprocessingData.VariableHasType<DateTime>(i)) {
[10711]314          double columnVariance = statisticsLogic.GetVariance(i);
[10820]315          if (columnVariance < variance) {
[10715]316            columns.Add(i);
[10711]317          }
318        }
319      }
[10715]320      return columns;
[10711]321    }
322
[10715]323    public void DeleteRowsWithMissingValuesGreater(double percent) {
324      DeleteRows(RowsWithMissingValuesGreater(percent));
325    }
326
327    public void DeleteColumnsWithMissingValuesGreater(double percent) {
328      DeleteColumns(ColumnsWithMissingValuesGreater(percent));
329    }
330
331    public void DeleteColumnsWithVarianceSmaller(double variance) {
332      DeleteColumns(ColumnsWithVarianceSmaller(variance));
333    }
334
[10737]335    private void DeleteRows(List<int> rows) {
336      rows.Sort();
337      rows.Reverse();
[10820]338      preprocessingData.InTransaction(() => {
339        foreach (int row in rows) {
[10715]340          preprocessingData.DeleteRow(row);
341        }
342      });
343    }
344
[10737]345    private void DeleteColumns(List<int> columns) {
346      columns.Sort();
347      columns.Reverse();
[10820]348      preprocessingData.InTransaction(() => {
349        foreach (int column in columns) {
[10715]350          preprocessingData.DeleteColumn(column);
351        }
352      });
353    }
[10249]354  }
[10193]355}
Note: See TracBrowser for help on using the repository browser.