source: branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/PreprocessingDataManipulation.cs @ 10235

using HeuristicLab.Data;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace HeuristicLab.DataPreprocessing.Implementations
{
    class PreprocessingDataManipulation
    {
        private IPreprocessingData preprocessingData;
        private StatisticInfo statisticInfo;

        public PreprocessingDataManipulation(IPreprocessingData _prepocessingData) {
            preprocessingData = _prepocessingData;
            statisticInfo = new StatisticInfo(preprocessingData);
        }

        public void ReplaceIndicesByValue<T>(string variableName, IEnumerable<int> indices, T value)
        {
            foreach (int index in indices)
            {
                preprocessingData.SetCell<T>(variableName, index, value);
            }
        }

        public void ReplaceIndicesByAverageValue(string variableName, IEnumerable<int> indices)
        {
            double average = statisticInfo.GetAverage(variableName);
            ReplaceIndicesByValue<double>(variableName, indices, average);
        }

        public void ReplaceIndicesByMedianValue(string variableName, IEnumerable<int> indices)
        {
            double median = statisticInfo.GetMedian(variableName);
            ReplaceIndicesByValue<double>(variableName, indices, median);
        }

        public void ReplaceIndicesByRandomValue(string variableName, IEnumerable<int> indices)
        {
            Random r = new Random();

            double max = statisticInfo.GetMax<double>(variableName);
            double min = statisticInfo.GetMin<double>(variableName);
            double randMultiplier = (max - min);
            foreach (int index in indices)
            {
                double rand = r.NextDouble() * randMultiplier + min;
                preprocessingData.SetCell<double>(variableName, index, rand);
            }
        }

        public void ReplaceIndicesByLinearInterpolationOfNeighbours(string variableName, IEnumerable<int> indices)
        {
            int countValues = preprocessingData.GetValues<double>(variableName).Count();
            foreach (int index in indices)
            {
                // dont replace first or last values
                if (index > 0 && index < countValues)
                {
                    int prevIndex = indexOfPrevPresentValue(variableName, index);
                    int nextIndex = indexOfNextPresentValue(variableName, index);

                    // no neighbours found
                    if (prevIndex < 0 && nextIndex >= countValues)
                    {
                        continue;
                    }
                    double prev = preprocessingData.GetCell<double>(variableName, prevIndex);
                    double next = preprocessingData.GetCell<double>(variableName, nextIndex);

                    int valuesToInterpolate = nextIndex - prevIndex;

                    double interpolationStep = (prev + next) / valuesToInterpolate;

                    for (int i = prevIndex; i < nextIndex; ++i) {
                        double interpolated = prev + (interpolationStep * (i-prevIndex));
                        preprocessingData.SetCell<double>(variableName, i, interpolated);
                    }
                }
            }
        }

        private int indexOfPrevPresentValue(string variableName, int start) {
            int offset = start - 1;
            while(offset >= 0 && preprocessingData.IsMissingValue(variableName, offset)){
                offset--;
            }

            return offset;
        }

        private int indexOfNextPresentValue(string variableName, int start)
        {
            int offset = start + 1;
            while (offset < preprocessingData.Rows && preprocessingData.IsMissingValue(variableName,  offset))
            {
                offset++;
            }

            return offset;
        }

        public void ReplaceIndicesByMostCommonValue(string variableName, IEnumerable<int> indices)
        {
            if (preprocessingData.IsType<double>(variableName))
            {
                ReplaceIndicesByValue<double>(variableName, indices, statisticInfo.GetMostCommonValue<double>(variableName));
            }
            else if (preprocessingData.IsType<string>(variableName))
            {
                ReplaceIndicesByValue<string>(variableName, indices, statisticInfo.GetMostCommonValue<string>(variableName));
            }
            else if (preprocessingData.IsType<DateTime>(variableName))
            {
                ReplaceIndicesByValue<DateTime>(variableName, indices, statisticInfo.GetMostCommonValue<DateTime>(variableName));
            }
            else
            {
                throw new ArgumentException("column with index: " + variableName + " contains a non supported type.");
            }
        }

        public void ShuffleWithRanges(IEnumerable<IntRange> ranges)
        {
            // init random outside loop
            Random random = new Random();

            // process all given ranges - e.g. TrainingPartition, Trainingpartition
            foreach (IntRange range in ranges) { 
                List<Tuple<int, int>> shuffledIndices = new List<Tuple<int,int>>();
               
                // generate random indices used for shuffeling each column
                for (int i = range.End; i > range.Start; --i)
                {
                    int rand = random.Next(range.Start, i);
                    shuffledIndices.Add(new Tuple<int,int>(i,rand));
                }

                foreach (string variableName in preprocessingData.VariableNames)
                {
                    if (preprocessingData.IsType<double>(variableName))
                    {
                        reOrderToIndices<double>(variableName, shuffledIndices);
                    }
                    else if (preprocessingData.IsType<string>(variableName))
                    {
                        reOrderToIndices<string>(variableName, shuffledIndices);
                    }
                    else if (preprocessingData.IsType<DateTime>(variableName))
                    {
                        reOrderToIndices<DateTime>(variableName, shuffledIndices);
                    }
                }
            }      
        }

        public void reOrderToIndices<T>(string variableName, List<Tuple<int, int>> indices) {
            // process all columns equally
            foreach(Tuple<int, int> index in indices)
            {
                int originalIndex = index.Item1;
                int replaceIndex = index.Item2;

                T tmp = preprocessingData.GetCell<T>(variableName, originalIndex);
                T replaceValue = preprocessingData.GetCell<T>(variableName, replaceIndex);

                preprocessingData.SetCell<T>(variableName, originalIndex, replaceValue);
                preprocessingData.SetCell<T>(variableName, replaceIndex, tmp);
            }
        }
    }
}