Free cookie consent management tool by TermsFeed Policy Generator

source: branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/ManipulationLogic.cs @ 10811

Last change on this file since 10811 was 10811, checked in by sbreuer, 10 years ago
  • provide default value (false) for considerSelection parameter
File size: 14.5 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using HeuristicLab.Data;
26
27namespace HeuristicLab.DataPreprocessing {
28  public class ManipulationLogic : IManipulationLogic {
29    private ITransactionalPreprocessingData preprocessingData;
30    private IStatisticsLogic statisticsLogic;
31    private ISearchLogic searchLogic;
32    private IDataGridLogic dataGridLogic;
33
34    public ManipulationLogic(ITransactionalPreprocessingData _prepocessingData, ISearchLogic theSearchLogic, IStatisticsLogic theStatisticsLogic, IDataGridLogic theDataGridLogic) {
35      preprocessingData = _prepocessingData;
36      searchLogic = theSearchLogic;
37      statisticsLogic = theStatisticsLogic;
38      dataGridLogic = theDataGridLogic;
39    }
40
41    public void ReplaceIndicesByValue<T>(int columnIndex, IEnumerable<int> rowIndices, T value) {
42      foreach (int index in rowIndices) {
43        preprocessingData.SetCell<T>(columnIndex, index, value);
44      }
45    }
46
47    public void ReplaceIndicesByAverageValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
48      preprocessingData.InTransaction(() => {
49        foreach (var column in cells) {
50          if (preprocessingData.IsType<double>(column.Key)) {
51            double average = statisticsLogic.GetAverage(column.Key, considerSelection);
52            ReplaceIndicesByValue<double>(column.Key, column.Value, average);
53          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
54            DateTime average = statisticsLogic.GetAverageDateTime(column.Key, considerSelection);
55            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, average);
56          }
57        }
58      });
59    }
60
61    public void ReplaceIndicesByMedianValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
62      preprocessingData.InTransaction(() => {
63        foreach (var column in cells) {
64          if (preprocessingData.IsType<double>(column.Key)) {
65            double median = statisticsLogic.GetMedian(column.Key, considerSelection);
66            ReplaceIndicesByValue<double>(column.Key, column.Value, median);
67          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
68            DateTime median = statisticsLogic.GetMedianDateTime(column.Key, considerSelection);
69            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, median);
70          }
71        }
72      });
73    }
74
75    public void ReplaceIndicesByRandomValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
76      preprocessingData.InTransaction(() => {
77        Random r = new Random();
78
79        foreach (var column in cells) {
80          if (preprocessingData.IsType<double>(column.Key)) {
81            double max = statisticsLogic.GetMax<double>(column.Key, considerSelection);
82            double min = statisticsLogic.GetMin<double>(column.Key, considerSelection);
83            double randMultiplier = (max - min);
84            foreach (int index in column.Value) {
85              double rand = r.NextDouble() * randMultiplier + min;
86              preprocessingData.SetCell<double>(column.Key, index, rand);
87            }
88          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
89            DateTime min = statisticsLogic.GetMin<DateTime>(column.Key, considerSelection);
90            DateTime max = statisticsLogic.GetMax<DateTime>(column.Key, considerSelection);
91            double randMultiplier = (max - min).TotalSeconds;
92            foreach (int index in column.Value) {
93              double rand = r.NextDouble() * randMultiplier;
94              preprocessingData.SetCell<DateTime>(column.Key, index, min.AddSeconds(rand));
95            }
96          }
97        }
98      });
99    }
100
101    public void ReplaceIndicesByLinearInterpolationOfNeighbours(IDictionary<int, IList<int>> cells) {
102      preprocessingData.InTransaction(() => {
103        foreach (var column in cells) {
104          int countValues = 0;
105          if (preprocessingData.IsType<double>(column.Key)) {
106            countValues = preprocessingData.GetValues<double>(column.Key).Count();
107          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
108            countValues = preprocessingData.GetValues<DateTime>(column.Key).Count();
109          }
110
111          foreach (int index in column.Value) {
112            // dont replace first or last values
113            if (index > 0 && index < countValues) {
114              int prevIndex = indexOfPrevPresentValue(column.Key, index);
115              int nextIndex = indexOfNextPresentValue(column.Key, index);
116
117              // no neighbours found
118              if (prevIndex < 0 && nextIndex >= countValues) {
119                continue;
120              }
121
122              int valuesToInterpolate = nextIndex - prevIndex;
123
124              if (preprocessingData.IsType<double>(column.Key)) {
125                double prev = preprocessingData.GetCell<double>(column.Key, prevIndex);
126                double next = preprocessingData.GetCell<double>(column.Key, nextIndex);
127                double interpolationStep = (next - prev) / valuesToInterpolate;
128
129                for (int i = prevIndex; i < nextIndex; ++i) {
130                  double interpolated = prev + (interpolationStep * (i - prevIndex));
131                  preprocessingData.SetCell<double>(column.Key, i, interpolated);
132                }
133              } else if (preprocessingData.IsType<DateTime>(column.Key)) {
134                DateTime prev = preprocessingData.GetCell<DateTime>(column.Key, prevIndex);
135                DateTime next = preprocessingData.GetCell<DateTime>(column.Key, nextIndex);
136                double interpolationStep = (next - prev).TotalSeconds / valuesToInterpolate;
137
138                for (int i = prevIndex; i < nextIndex; ++i) {
139                  DateTime interpolated = prev.AddSeconds(interpolationStep * (i - prevIndex));
140                  preprocessingData.SetCell<DateTime>(column.Key, i, interpolated);
141                }
142              }
143            }
144          }
145        }
146      });
147    }
148
149    private int indexOfPrevPresentValue(int columnIndex, int start) {
150      int offset = start - 1;
151      while (offset >= 0 && searchLogic.IsMissingValue(columnIndex, offset)) {
152        offset--;
153      }
154
155      return offset;
156    }
157
158    private int indexOfNextPresentValue(int columnIndex, int start) {
159      int offset = start + 1;
160      while (offset < preprocessingData.Rows && searchLogic.IsMissingValue(columnIndex, offset)) {
161        offset++;
162      }
163
164      return offset;
165    }
166
167    public void ReplaceIndicesByMostCommonValue(IDictionary<int, IList<int>> cells, bool considerSelection) {
168      preprocessingData.InTransaction(() => {
169        foreach (var column in cells) {
170          if (preprocessingData.IsType<double>(column.Key)) {
171            ReplaceIndicesByValue<double>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<double>(column.Key, considerSelection));
172          } else if (preprocessingData.IsType<string>(column.Key)) {
173            ReplaceIndicesByValue<string>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<string>(column.Key, considerSelection));
174          } else if (preprocessingData.IsType<DateTime>(column.Key)) {
175            ReplaceIndicesByValue<DateTime>(column.Key, column.Value, statisticsLogic.GetMostCommonValue<DateTime>(column.Key, considerSelection));
176          } else {
177            throw new ArgumentException("column with index: " + column.Key + " contains a non supported type.");
178          }
179        }
180      });
181    }
182
183    public void ShuffleWithRanges() {
184      ShuffleWithRanges(new[] {
185        preprocessingData.TestPartition,
186        preprocessingData.TrainingPartition
187      });
188    }
189
190    public void ShuffleWithRanges(IEnumerable<IntRange> ranges) {
191      // init random outside loop
192      Random random = new Random();
193
194      preprocessingData.InTransaction(() => {
195        // process all given ranges - e.g. TrainingPartition, TestPartition
196        foreach (IntRange range in ranges) {
197          List<Tuple<int, int>> shuffledIndices = new List<Tuple<int, int>>();
198
199          // generate random indices used for shuffeling each column
200          for (int i = range.End - 1; i >= range.Start; --i) {
201            int rand = random.Next(range.Start, i);
202            shuffledIndices.Add(new Tuple<int, int>(i, rand));
203          }
204
205          ShuffleToIndices(shuffledIndices);
206        }
207      });
208    }
209
210    public void ReOrderToIndices(IEnumerable<int> indices) {
211      List<Tuple<int, int>> indicesTuple = new List<Tuple<int, int>>();
212
213      for (int i = 0; i < indices.Count(); ++i) {
214        indicesTuple.Add(new Tuple<int, int>(i, indices.ElementAt(i)));
215      }
216
217      ReOrderToIndices(indicesTuple);
218    }
219
220    public void ReOrderToIndices(IList<System.Tuple<int, int>> indices) {
221      preprocessingData.InTransaction(() => {
222        for (int i = 0; i < preprocessingData.Columns; ++i) {
223          if (preprocessingData.IsType<double>(i)) {
224            reOrderToIndices<double>(i, indices);
225          } else if (preprocessingData.IsType<string>(i)) {
226            reOrderToIndices<string>(i, indices);
227          } else if (preprocessingData.IsType<DateTime>(i)) {
228            reOrderToIndices<DateTime>(i, indices);
229          }
230        }
231      });
232    }
233
234    public void ShuffleToIndices(IList<System.Tuple<int, int>> indices)
235    {
236      preprocessingData.InTransaction(() =>
237      {
238        for (int i = 0; i < preprocessingData.Columns; ++i)
239        {
240          if (preprocessingData.IsType<double>(i))
241          {
242            ShuffleToIndices<double>(i, indices);
243          }
244          else if (preprocessingData.IsType<string>(i))
245          {
246            ShuffleToIndices<string>(i, indices);
247          }
248          else if (preprocessingData.IsType<DateTime>(i))
249          {
250            ShuffleToIndices<DateTime>(i, indices);
251          }
252        }
253      });
254    }
255
256    private void reOrderToIndices<T>(int columnIndex, IList<Tuple<int, int>> indices) {
257
258      List<T> originalData = new List<T>(preprocessingData.GetValues<T>(columnIndex));
259
260      // process all columns equally
261      foreach (Tuple<int, int> index in indices) {
262        int originalIndex = index.Item1;
263        int replaceIndex = index.Item2;
264
265        T replaceValue = originalData.ElementAt<T>(replaceIndex);
266        preprocessingData.SetCell<T>(columnIndex, originalIndex, replaceValue);
267      }
268    }
269
270    private void ShuffleToIndices<T>(int columnIndex, IList<Tuple<int, int>> indices)
271    {
272      // process all columns equally
273      foreach (Tuple<int, int> index in indices)
274      {
275        int originalIndex = index.Item1;
276        int replaceIndex = index.Item2;
277
278        T tmp = preprocessingData.GetCell<T>(columnIndex, originalIndex);
279        T replaceValue = preprocessingData.GetCell<T>(columnIndex, replaceIndex);
280
281        preprocessingData.SetCell<T>(columnIndex, originalIndex, replaceValue);
282        preprocessingData.SetCell<T>(columnIndex, replaceIndex, tmp);
283      }
284    }
285
286    public void ReplaceIndicesByValue(IDictionary<int, IList<int>> cells, string value) {
287      preprocessingData.InTransaction(() => {
288        foreach (var column in cells) {
289          foreach (var rowIdx in column.Value) {
290            dataGridLogic.SetValue(value, column.Key, rowIdx);
291          }
292        }
293      });
294    }
295
296
297    public List<int> RowsWithMissingValuesGreater(double percent) {
298
299      List<int> rows= new List<int>();
300
301      for (int i = 0; i < preprocessingData.Rows; ++i)
302      {
303        int missingCount = statisticsLogic.GetRowMissingValueCount(i);
304        if (100f / preprocessingData.Columns * missingCount > percent)
305        {
306          rows.Add(i);
307        }
308      }
309
310      return rows;
311    }
312
313    public List<int> ColumnsWithMissingValuesGreater(double percent) {
314
315      List<int> columns = new List<int>();
316      for (int i = 0; i < preprocessingData.Columns; ++i) {
317        int missingCount = statisticsLogic.GetMissingValueCount(i);
318        if (100f / preprocessingData.Rows * missingCount > percent) {
319          columns.Add(i);
320        }
321      }
322
323      return columns;
324    }
325
326    public List<int> ColumnsWithVarianceSmaller(double variance) {
327
328      List<int> columns = new List<int>();
329      for (int i = 0; i < preprocessingData.Columns; ++i) {
330        if (preprocessingData.IsType<double>(i) || preprocessingData.IsType<DateTime>(i))
331        {
332          double columnVariance = statisticsLogic.GetVariance(i);
333          if (columnVariance < variance)
334          {
335            columns.Add(i);
336          }
337        }
338      }
339      return columns;
340    }
341
342    public void DeleteRowsWithMissingValuesGreater(double percent) {
343      DeleteRows(RowsWithMissingValuesGreater(percent));
344    }
345
346    public void DeleteColumnsWithMissingValuesGreater(double percent) {
347      DeleteColumns(ColumnsWithMissingValuesGreater(percent));
348    }
349
350    public void DeleteColumnsWithVarianceSmaller(double variance) {
351      DeleteColumns(ColumnsWithVarianceSmaller(variance));
352    }
353
354    private void DeleteRows(List<int> rows) {
355      rows.Sort();
356      rows.Reverse();
357      preprocessingData.InTransaction(() =>
358      {
359        foreach (int row in rows)
360        {
361          preprocessingData.DeleteRow(row);
362        }
363      });
364    }
365
366    private void DeleteColumns(List<int> columns) {
367      columns.Sort();
368      columns.Reverse();
369      preprocessingData.InTransaction(() =>
370      {
371        foreach (int column in columns)
372        {
373          preprocessingData.DeleteColumn(column);
374        }
375      });
376    }
377
378    public event DataPreprocessingChangedEventHandler Changed {
379      add { dataGridLogic.Changed += value; }
380      remove { dataGridLogic.Changed -= value; }
381    }
382  }
383}
Note: See TracBrowser for help on using the repository browser.