source: branches/DataPreprocessing/HeuristicLab.DataPreprocessing/3.3/Implementations/PreprocessingData.cs @ 10192

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Problems.DataAnalysis;

namespace HeuristicLab.DataPreprocessing {
  [Item("PreprocessingData", "Represents data used for preprocessing.")]
  public class PreprocessingData : NamedItem, IPreprocessingData {

    private IDictionary<string, IList> variableValues;

    private IList<string> variableNames;

    private IDictionary<string, int> variableNameIndices;

    private double trainingToTestRatio;
    private StatisticInfo statisticInfo;

    private PreprocessingData(PreprocessingData original, Cloner cloner)
      : base(original, cloner) {
      variableValues = new Dictionary<string, IList>(variableValues);
      variableNameIndices = new Dictionary<string, int>(variableNameIndices);
    }

    public PreprocessingData(IDataAnalysisProblemData problemData)
      : base() {
      Name = "-";

      variableNames = new List<string>(problemData.Dataset.VariableNames);
      // create dictionary from variable name to index
      variableNameIndices = new Dictionary<string, int>();
      var variableNamesList = problemData.Dataset.VariableNames.ToList();
      for (int i = 0; i < variableNamesList.Count; i++) {
        variableNameIndices.Add(variableNamesList[i], i);
      }

      // copy values
      variableValues = new Dictionary<string, IList>();
      foreach (var variableName in problemData.Dataset.VariableNames) {
        if (problemData.Dataset.IsType<double>(variableName)) {
          variableValues[variableName] = problemData.Dataset.GetDoubleValues(variableName).ToList();
        } else if (problemData.Dataset.IsType<string>(variableName)) {
          variableValues[variableName] = CreateColumn<string>(problemData.Dataset, variableNameIndices[variableName], x => x);
        } else if (problemData.Dataset.IsType<DateTime>(variableName)) {
          variableValues[variableName] = CreateColumn<DateTime>(problemData.Dataset, variableNameIndices[variableName], x => DateTime.Parse(x));
        } else {
          throw new ArgumentException("The datatype of column " + variableName + " must be of type List<double>, List<string> or List<DateTime>");
        }
      }

      trainingToTestRatio = (double)problemData.TrainingPartition.Size / problemData.TestPartition.Size;
      Columns = problemData.Dataset.Columns;
      Rows = problemData.Dataset.Rows;

      statisticInfo = new StatisticInfo(this);
    }

    private static IList CreateColumn<T>(Dataset ds, int column, Func<string, T> selector) {
      var list = new List<T>(ds.Rows);
      for (int row = 0; row < ds.Rows; row++) {
        list[row] = selector(ds.GetValue(row, column));
      }
      return list;
    }

    #region NamedItem abstract Member Implementations

    public override IDeepCloneable Clone(Cloner cloner) {
      return new PreprocessingData(this, cloner);
    }

    #endregion

    #region IPreprocessingData Members

    public T GetCell<T>(string variableName, int row) {
      return (T)variableValues[variableName][row];
    }

    public void SetCell<T>(string variableName, int row, T value) {
      variableValues[variableName][row] = value;
    }

    public IEnumerable<T> GetValues<T>(string variableName) {
        return (IEnumerable<T>)variableValues[variableName];
    }

    public void SetValues<T>(string variableName, IEnumerable<T> values) {
      variableValues[variableName] = values.ToList();
    }

    public void InsertRow(int rowIndex) {
      throw new NotImplementedException();
    }

    public void DeleteRow(int rowIndex) {
      throw new NotImplementedException();
    }

    public void InsertColumn(string variableName, int columnIndex) {
      throw new NotImplementedException();
    }

    public void DeleteColumn(string variableName) {
      throw new NotImplementedException();
    }

    public IEnumerable<string> VariableNames {
      get { return variableNames; }
    }

    public bool IsType<T>(string variableName) {
      return variableValues[variableName] is List<T>;
    }

    public int Columns {
      get;
      private set;
    }

    public int Rows {
      get;
      private set;
    }

    public void ExportTo(IDataAnalysisProblemData problemData) {
      throw new NotImplementedException();
    }

    public IDictionary<string, IEnumerable<int>> GetMissingValueIndices() {
      var dic = new Dictionary<string, IEnumerable<int>>();
      foreach (string variableName in VariableNames) {
        dic.Add(variableName, GetMissingValueIndices(variableName));
      }
      return dic;
    }

    public bool IsMissingValue(string variableName, int rowIndex) {
      if (IsType<double>(variableName)) {
        return double.IsNaN(GetCell<double>(variableName, rowIndex));
      } else if (IsType<string>(variableName)) {
        return string.IsNullOrEmpty(GetCell<string>(variableName, rowIndex));
      } else if (IsType<DateTime>(variableName)) {
        return GetCell<DateTime>(variableName, rowIndex).Equals(DateTime.MinValue);
      } else {
        throw new ArgumentException("cell in column with variableName: " + variableName + " and row index " + rowIndex + " contains a non supported type.");
      }
    }

    public IEnumerable<int> GetMissingValueIndices(string variableName) {
      if (IsType<double>(variableName)) {
        return GetValues<double>(variableName).Select((s, i) => new { i, s }).Where(t => double.IsNaN(t.s)).Select(t => t.i);
      } else if (IsType<string>(variableName)) {
        return GetValues<string>(variableName).Select((s, i) => new { i, s }).Where(t => string.IsNullOrEmpty(t.s)).Select(t => t.i);
      } else if (IsType<DateTime>(variableName)) {
          return GetValues<DateTime>(variableName).Select((s, i) => new { i, s }).Where(t => t.s.Equals(DateTime.MinValue)).Select(t => t.i);
      } else {
        throw new ArgumentException("column with variableName: " + variableName + " contains a non supported type.");
      }
    }

    public void ReplaceIndicesByValue<T>(string variableName, IEnumerable<int> indices, T value)
    {
        foreach (int index in indices)
        {
            SetCell<T>(variableName, index, value);
        }
    }

    public void ReplaceIndicesByAverageValue(string variableName, IEnumerable<int> indices) {
        double average = statisticInfo.GetAverage(variableName);
        ReplaceIndicesByValue<double>(variableName, indices, average);
    }

    public void ReplaceIndicesByAverageValue(string variableName, IEnumerable<int> indices) {
        double median = statisticInfo.GetMedian(variableName);
        ReplaceIndicesByValue<double>(variableName, indices, median);
    }

    public void ReplaceIndicesByRandomValue(string variableName, IEnumerable<int> indices) {
        Random r = new Random();

        double max = statisticInfo.GetMax<double>(variableName);
        double min = statisticInfo.GetMin<double>(variableName);
        double randMultiplier = (max - min);
        foreach (int index in indices)
        {
            double rand = r.NextDouble() * randMultiplier + min;
            SetCell<double>(variableName, index, rand);
        } 
    }

    public void ReplaceIndicesByLinearInterpolationOfNeighbours(string variableName, IEnumerable<int> indices) {
        int countValues = GetValues<double>(variableName).Count();
        foreach (int index in indices)
        {
            // dont replace first or last values
            if (index > 0 && index < countValues) {
               double prev = GetCell<double>(variableName, index - 1);
               double next = GetCell<double>(variableName, index + 1);

               double interpolated = (prev + next) / 2;

               SetCell<double>(variableName, index, interpolated);
            }
        } 
    }

    public void ReplaceIndicesByMostCommonValue(string variableName, IEnumerable<int> indices) {   
      if (IsType<double>(variableName)) {
        ReplaceIndicesByValue<double>(variableName, indices,statisticInfo.GetMostCommonValue<double>(variableName));
      } else if (IsType<string>(variableName)) {
        ReplaceIndicesByValue<string>(variableName, indices, statisticInfo.GetMostCommonValue<string>(variableName));
      } else if (IsType<DateTime>(variableName)) {
        ReplaceIndicesByValue<DateTime>(variableName, indices, statisticInfo.GetMostCommonValue<DateTime>(variableName));
      } else {
        throw new ArgumentException("column with index: " + variableName + " contains a non supported type.");
      }
    }

    #endregion
  }
}