source: branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/PreprocessingDataManipulation.cs @ 10355

using System;
using System.Collections.Generic;
using System.Linq;
using HeuristicLab.Data;
using System.Collections;

namespace HeuristicLab.DataPreprocessing {
  public class PreprocessingDataManipulation : IPreprocessingDataManipulation {
    private IPreprocessingData preprocessingData;
    private IStatisticsLogic statisticInfo;
    private ISearchLogic searchLogic;

    public PreprocessingDataManipulation(IPreprocessingData _prepocessingData, ISearchLogic theSearchLogic, IStatisticsLogic theStatisticsLogic) {
      preprocessingData = _prepocessingData;
      searchLogic = theSearchLogic;
      statisticInfo = theStatisticsLogic;
    }

    public void ReplaceIndicesByValue<T>(string variableName, IEnumerable<int> indices, T value) {
      foreach (int index in indices) {
        preprocessingData.SetCell<T>(variableName, index, value);
      }
    }

    public void ReplaceIndicesByAverageValue(string variableName, IEnumerable<int> indices) {
      double average = statisticInfo.GetAverage(variableName);
      ReplaceIndicesByValue<double>(variableName, indices, average);
    }

    public void ReplaceIndicesByMedianValue(string variableName, IEnumerable<int> indices) {
      double median = statisticInfo.GetMedian(variableName);
      ReplaceIndicesByValue<double>(variableName, indices, median);
    }

    public void ReplaceIndicesByRandomValue(string variableName, IEnumerable<int> indices) {
      Random r = new Random();

      double max = statisticInfo.GetMax<double>(variableName);
      double min = statisticInfo.GetMin<double>(variableName);
      double randMultiplier = (max - min);
      foreach (int index in indices) {
        double rand = r.NextDouble() * randMultiplier + min;
        preprocessingData.SetCell<double>(variableName, index, rand);
      }
    }

    public void ReplaceIndicesByLinearInterpolationOfNeighbours(string variableName, IEnumerable<int> indices) {
      int countValues = preprocessingData.GetValues<double>(variableName).Count();
      foreach (int index in indices) {
        // dont replace first or last values
        if (index > 0 && index < countValues) {
          int prevIndex = indexOfPrevPresentValue(variableName, index);
          int nextIndex = indexOfNextPresentValue(variableName, index);

          // no neighbours found
          if (prevIndex < 0 && nextIndex >= countValues) {
            continue;
          }
          double prev = preprocessingData.GetCell<double>(variableName, prevIndex);
          double next = preprocessingData.GetCell<double>(variableName, nextIndex);

          int valuesToInterpolate = nextIndex - prevIndex;

          double interpolationStep = (prev + next) / valuesToInterpolate;

          for (int i = prevIndex; i < nextIndex; ++i) {
            double interpolated = prev + (interpolationStep * (i - prevIndex));
            preprocessingData.SetCell<double>(variableName, i, interpolated);
          }
        }
      }
    }

    private int indexOfPrevPresentValue(string variableName, int start) {
      int offset = start - 1;
      while (offset >= 0 && searchLogic.IsMissingValue(variableName, offset)) {
        offset--;
      }

      return offset;
    }

    private int indexOfNextPresentValue(string variableName, int start) {
      int offset = start + 1;
      while (offset < preprocessingData.Rows && searchLogic.IsMissingValue(variableName, offset)) {
        offset++;
      }

      return offset;
    }

    public void ReplaceIndicesByMostCommonValue(string variableName, IEnumerable<int> indices) {
      if (preprocessingData.IsType<double>(variableName)) {
        ReplaceIndicesByValue<double>(variableName, indices, statisticInfo.GetMostCommonValue<double>(variableName));
      } else if (preprocessingData.IsType<string>(variableName)) {
        ReplaceIndicesByValue<string>(variableName, indices, statisticInfo.GetMostCommonValue<string>(variableName));
      } else if (preprocessingData.IsType<DateTime>(variableName)) {
        ReplaceIndicesByValue<DateTime>(variableName, indices, statisticInfo.GetMostCommonValue<DateTime>(variableName));
      } else {
        throw new ArgumentException("column with index: " + variableName + " contains a non supported type.");
      }
    }

    public void ShuffleWithRanges(IEnumerable<IntRange> ranges) {
      // init random outside loop
      Random random = new Random();

      // process all given ranges - e.g. TrainingPartition, Trainingpartition
      foreach (IntRange range in ranges) {
        List<Tuple<int, int>> shuffledIndices = new List<Tuple<int, int>>();

        // generate random indices used for shuffeling each column
        for (int i = range.End; i > range.Start; --i) {
          int rand = random.Next(range.Start, i);
          shuffledIndices.Add(new Tuple<int, int>(i, rand));
        }

        reOrderToIndices(shuffledIndices);
      }
    }

    public void reOrderToIndices(IEnumerable<int> indices) {
      List<Tuple<int, int>> indicesTuple = new List<Tuple<int, int>>();

      for (int i = 0; i < indices.Count(); ++i) {
        indicesTuple.Add(new Tuple<int, int>(i, indices.ElementAt(i)));
      }

      reOrderToIndices(indicesTuple);
    }

    public void reOrderToIndices(IList<System.Tuple<int, int>> indices) {
      foreach (string variableName in preprocessingData.VariableNames) {
        if (preprocessingData.IsType<double>(variableName)) {
          reOrderToIndices<double>(variableName, indices);
        } else if (preprocessingData.IsType<string>(variableName)) {
          reOrderToIndices<string>(variableName, indices);
        } else if (preprocessingData.IsType<DateTime>(variableName)) {
          reOrderToIndices<DateTime>(variableName, indices);
        }
      }
    }

    private void reOrderToIndices<T>(string variableName, IList<Tuple<int, int>> indices) {

      List<T> originalData = new List<T>(preprocessingData.GetValues<T>(variableName));

      // process all columns equally
      foreach (Tuple<int, int> index in indices) {
        int originalIndex = index.Item1;
        int replaceIndex = index.Item2;

        T replaceValue = originalData.ElementAt<T>(replaceIndex);
        preprocessingData.SetCell<T>(variableName, originalIndex, replaceValue);
      }
    }
  }
}