source: trunk/sources/HeuristicLab.DataPreprocessing/3.4/Data/TransactionalPreprocessingData.cs @ 13517

Last change on this file since 13517 was 13517, checked in by pfleck, 5 years ago

#2559

  • Correctly adapt training and test partition when inserting and deleting rows.
  • Fixed problem with correlation matrix.
  • Fixed bug within value frequency calculation for histogram.
File size: 14.6 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections;
24using System.Collections.Generic;
25using System.Globalization;
26using System.Linq;
27using HeuristicLab.Common;
28using HeuristicLab.Core;
29using HeuristicLab.Data;
30using HeuristicLab.Problems.DataAnalysis;
31
32namespace HeuristicLab.DataPreprocessing {
33  [Item("PreprocessingData", "Represents data used for preprocessing.")]
34  public class TransactionalPreprocessingData : PreprocessingData, ITransactionalPreprocessingData {
35
36    private class Snapshot {
37      public IList<IList> VariableValues { get; set; }
38      public IList<string> VariableNames { get; set; }
39
40      public IntRange TrainingPartition { get; set; }
41      public IntRange TestPartition { get; set; }
42      public IList<ITransformation> Transformations { get; set; }
43      public DataPreprocessingChangedEventType ChangedType { get; set; }
44
45      public int ChangedColumn { get; set; }
46      public int ChangedRow { get; set; }
47    }
48
49    private const int MAX_UNDO_DEPTH = 5;
50
51    private readonly IList<Snapshot> undoHistory = new List<Snapshot>();
52    private readonly Stack<DataPreprocessingChangedEventType> eventStack = new Stack<DataPreprocessingChangedEventType>();
53
54    public bool IsInTransaction { get { return eventStack.Count > 0; } }
55
56    public TransactionalPreprocessingData(IDataAnalysisProblemData problemData)
57      : base(problemData) {
58    }
59
60    protected TransactionalPreprocessingData(TransactionalPreprocessingData original, Cloner cloner)
61      : base(original, cloner) {
62    }
63
64    private void SaveSnapshot(DataPreprocessingChangedEventType changedType, int column, int row) {
65      if (IsInTransaction) return;
66
67      var currentSnapshot = new Snapshot {
68        VariableValues = CopyVariableValues(variableValues),
69        VariableNames = new List<string>(variableNames),
70        TrainingPartition = new IntRange(TrainingPartition.Start, TrainingPartition.End),
71        TestPartition = new IntRange(TestPartition.Start, TestPartition.End),
72        Transformations = new List<ITransformation>(transformations),
73        ChangedType = changedType,
74        ChangedColumn = column,
75        ChangedRow = row
76      };
77
78      if (undoHistory.Count >= MAX_UNDO_DEPTH)
79        undoHistory.RemoveAt(0);
80
81      undoHistory.Add(currentSnapshot);
82    }
83
84    #region NamedItem abstract Member Implementations
85
86    public override IDeepCloneable Clone(Cloner cloner) {
87      return new TransactionalPreprocessingData(this, cloner);
88    }
89
90    #endregion
91
92    #region Overridden IPreprocessingData Members
93
94    public override T GetCell<T>(int columnIndex, int rowIndex) {
95      return (T)variableValues[columnIndex][rowIndex];
96    }
97
98    public override void SetCell<T>(int columnIndex, int rowIndex, T value) {
99      SaveSnapshot(DataPreprocessingChangedEventType.ChangeItem, columnIndex, rowIndex);
100
101      for (int i = Rows; i <= rowIndex; i++)
102        InsertRow(i);
103      for (int i = Columns; i <= columnIndex; i++)
104        InsertColumn<T>(i.ToString(), i);
105
106      variableValues[columnIndex][rowIndex] = value;
107      if (!IsInTransaction)
108        OnChanged(DataPreprocessingChangedEventType.ChangeItem, columnIndex, rowIndex);
109    }
110
111    public override string GetCellAsString(int columnIndex, int rowIndex) {
112      return variableValues[columnIndex][rowIndex].ToString();
113    }
114
115    public override string GetVariableName(int columnIndex) {
116      return variableNames[columnIndex];
117    }
118
119    public override int GetColumnIndex(string variableName) {
120      return variableNames.IndexOf(variableName);
121    }
122
123    public override bool VariableHasType<T>(int columnIndex) {
124      return columnIndex >= variableValues.Count || variableValues[columnIndex] is List<T>;
125    }
126
127    [Obsolete("use the index based variant, is faster")]
128    public override IList<T> GetValues<T>(string variableName, bool considerSelection) {
129      return GetValues<T>(GetColumnIndex(variableName), considerSelection);
130    }
131
132    public override IList<T> GetValues<T>(int columnIndex, bool considerSelection) {
133      if (considerSelection) {
134        var list = new List<T>();
135        foreach (var rowIdx in selection[columnIndex]) {
136          list.Add((T)variableValues[columnIndex][rowIdx]);
137        }
138        return list;
139      } else {
140        return (IList<T>)variableValues[columnIndex];
141      }
142    }
143
144    public override void SetValues<T>(int columnIndex, IList<T> values) {
145      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
146      if (VariableHasType<T>(columnIndex)) {
147        variableValues[columnIndex] = (IList)values;
148      } else {
149        throw new ArgumentException("The datatype of column " + columnIndex + " must be of type " + variableValues[columnIndex].GetType().Name + " but was " + typeof(T).Name);
150      }
151      if (!IsInTransaction)
152        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
153    }
154
155    public override bool SetValue(string value, int columnIndex, int rowIndex) {
156      bool valid = false;
157      if (VariableHasType<double>(columnIndex)) {
158        double val;
159        valid = double.TryParse(value, out val);
160        SetValueIfValid(columnIndex, rowIndex, valid, val);
161      } else if (VariableHasType<string>(columnIndex)) {
162        valid = value != null;
163        SetValueIfValid(columnIndex, rowIndex, valid, value);
164      } else if (VariableHasType<DateTime>(columnIndex)) {
165        DateTime date;
166        valid = DateTime.TryParse(value, out date);
167        SetValueIfValid(columnIndex, rowIndex, valid, date);
168      } else {
169        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
170      }
171
172      if (!IsInTransaction)
173        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
174
175      return valid;
176    }
177
178    public override bool Validate(string value, out string errorMessage, int columnIndex) {
179      if (columnIndex < 0 || columnIndex > VariableNames.Count()) {
180        throw new ArgumentOutOfRangeException("column index is out of range");
181      }
182
183      bool valid = false;
184      errorMessage = string.Empty;
185      if (VariableHasType<double>(columnIndex)) {
186        double val;
187        valid = double.TryParse(value, out val);
188        if (!valid) {
189          errorMessage = "Invalid Value (Valid Value Format: \"" + FormatPatterns.GetDoubleFormatPattern() + "\")";
190        }
191      } else if (VariableHasType<string>(columnIndex)) {
192        valid = value != null;
193        if (!valid) {
194          errorMessage = "Invalid Value (string must not be null)";
195        }
196      } else if (VariableHasType<DateTime>(columnIndex)) {
197        DateTime date;
198        valid = DateTime.TryParse(value, out date);
199        if (!valid) {
200          errorMessage = "Invalid Value (Valid Value Format: \"" + CultureInfo.CurrentCulture.DateTimeFormat + "\"";
201        }
202      } else {
203        throw new ArgumentException("column " + columnIndex + " contains a non supported type.");
204      }
205
206      return valid;
207    }
208
209    private void SetValueIfValid<T>(int columnIndex, int rowIndex, bool valid, T value) {
210      if (valid)
211        SetCell<T>(columnIndex, rowIndex, value);
212    }
213
214    public override bool AreAllStringColumns(IEnumerable<int> columnIndices) {
215      return columnIndices.All(x => VariableHasType<string>(x));
216    }
217
218
219    public override void InsertRow(int rowIndex) {
220      SaveSnapshot(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
221      foreach (IList column in variableValues) {
222        Type type = column.GetType().GetGenericArguments()[0];
223        column.Insert(rowIndex, type.IsValueType ? Activator.CreateInstance(type) : null);
224      }
225      if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
226        TrainingPartition.End++;
227        if (TrainingPartition.End <= TestPartition.Start) {
228          TestPartition.Start++;
229          TestPartition.End++;
230        }
231      } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
232        TestPartition.End++;
233        if (TestPartition.End <= TrainingPartition.Start) {
234          TestPartition.Start++;
235          TestPartition.End++;
236        }
237      }
238      if (!IsInTransaction)
239        OnChanged(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
240    }
241    public override void DeleteRow(int rowIndex) {
242      SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, rowIndex);
243      foreach (IList column in variableValues) {
244        column.RemoveAt(rowIndex);
245      }
246      if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
247        TrainingPartition.End--;
248        if (TrainingPartition.End <= TestPartition.Start) {
249          TestPartition.Start--;
250          TestPartition.End--;
251        }
252      } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
253        TestPartition.End--;
254        if (TestPartition.End <= TrainingPartition.Start) {
255          TestPartition.Start--;
256          TestPartition.End--;
257        }
258      }
259      if (!IsInTransaction)
260        OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, rowIndex);
261    }
262    public override void DeleteRowsWithIndices(IEnumerable<int> rows) {
263      SaveSnapshot(DataPreprocessingChangedEventType.AddRow, -1, -1);
264      foreach (int rowIndex in rows.OrderByDescending(x => x)) {
265        foreach (IList column in variableValues) {
266          column.RemoveAt(rowIndex);
267        }
268        if (TrainingPartition.Start <= rowIndex && rowIndex <= TrainingPartition.End) {
269          TrainingPartition.End--;
270          if (TrainingPartition.End <= TestPartition.Start) {
271            TestPartition.Start--;
272            TestPartition.End--;
273          }
274        } else if (TestPartition.Start <= rowIndex && rowIndex <= TestPartition.End) {
275          TestPartition.End--;
276          if (TestPartition.End <= TrainingPartition.Start) {
277            TestPartition.Start--;
278            TestPartition.End--;
279          }
280        }
281      }
282      if (!IsInTransaction)
283        OnChanged(DataPreprocessingChangedEventType.DeleteRow, -1, -1);
284    }
285
286    public override void InsertColumn<T>(string variableName, int columnIndex) {
287      SaveSnapshot(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
288      variableValues.Insert(columnIndex, new List<T>(Enumerable.Repeat(default(T), Rows)));
289      variableNames.Insert(columnIndex, variableName);
290      if (!IsInTransaction)
291        OnChanged(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
292    }
293
294    public override void DeleteColumn(int columnIndex) {
295      SaveSnapshot(DataPreprocessingChangedEventType.AddColumn, columnIndex, -1);
296      variableValues.RemoveAt(columnIndex);
297      variableNames.RemoveAt(columnIndex);
298      if (!IsInTransaction)
299        OnChanged(DataPreprocessingChangedEventType.DeleteColumn, columnIndex, -1);
300    }
301
302    public override void RenameColumn(int columnIndex, string name) {
303      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, columnIndex, -1);
304      if (columnIndex < 0 || columnIndex > variableNames.Count)
305        throw new ArgumentOutOfRangeException("columnIndex");
306      variableNames[columnIndex] = name;
307
308      if (!IsInTransaction)
309        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
310    }
311
312    public override void RenameColumns(IList<string> names) {
313      if (names == null) throw new ArgumentNullException("names");
314      if (names.Count != variableNames.Count) throw new ArgumentException("number of names must match the number of columns.", "names");
315
316      SaveSnapshot(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
317      for (int i = 0; i < names.Count; i++)
318        variableNames[i] = names[i];
319
320      if (!IsInTransaction)
321        OnChanged(DataPreprocessingChangedEventType.ChangeColumn, -1, -1);
322    }
323
324    public override Dataset ExportToDataset() {
325      IList<IList> values = new List<IList>();
326
327      for (int i = 0; i < Columns; ++i) {
328        values.Add(variableValues[i]);
329      }
330
331      var dataset = new Dataset(variableNames, values);
332      return dataset;
333    }
334
335    public override void ClearSelection() {
336      Selection = new Dictionary<int, IList<int>>();
337    }
338
339    public override event EventHandler SelectionChanged;
340
341    protected override void OnSelectionChanged() {
342      var listeners = SelectionChanged;
343      if (listeners != null) listeners(this, EventArgs.Empty);
344    }
345    #endregion
346
347    #region TransactionalPreprocessingData members
348
349    public bool IsUndoAvailable {
350      get { return undoHistory.Count > 0; }
351    }
352
353    public void Undo() {
354      if (IsUndoAvailable) {
355        Snapshot previousSnapshot = undoHistory[undoHistory.Count - 1];
356        variableValues = previousSnapshot.VariableValues;
357        variableNames = previousSnapshot.VariableNames;
358        TrainingPartition = previousSnapshot.TrainingPartition;
359        TestPartition = previousSnapshot.TestPartition;
360        transformations = previousSnapshot.Transformations;
361        undoHistory.Remove(previousSnapshot);
362        OnChanged(previousSnapshot.ChangedType,
363          previousSnapshot.ChangedColumn,
364          previousSnapshot.ChangedRow);
365      }
366    }
367
368    public void InTransaction(Action action, DataPreprocessingChangedEventType type = DataPreprocessingChangedEventType.Any) {
369      BeginTransaction(type);
370      action();
371      EndTransaction();
372    }
373
374    public void BeginTransaction(DataPreprocessingChangedEventType type) {
375      SaveSnapshot(type, -1, -1);
376      eventStack.Push(type);
377    }
378
379    public void EndTransaction() {
380      if (eventStack.Count == 0)
381        throw new InvalidOperationException("There is no open transaction that can be ended.");
382
383      var @event = eventStack.Pop();
384      OnChanged(@event, -1, -1);
385    }
386
387    #endregion
388  }
389}
Note: See TracBrowser for help on using the repository browser.