source: branches/3040_VectorBasedGP/HeuristicLab.Problems.Instances.DataAnalysis/3.3/Classification/TimeSeries/TimeSeriesInstanceProvider.cs @ 17418

#region License Information
/* HeuristicLab
 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.IO.Compression;
using System.Linq;
using HeuristicLab.Problems.DataAnalysis;

namespace HeuristicLab.Problems.Instances.DataAnalysis {
  public abstract class TimeSeriesInstanceProvider : ResourceClassificationInstanceProvider {
    //public override string Name {
    //  get { return "TimeSeries (Univariate) Problems"; }
    //}
    public override string Description {
      get { return "UEA & UCR TimeSeries Problems"; }
    }
    public override Uri WebLink {
      get { return new Uri("http://www.timeseriesclassification.com/"); }
    }
    public override string ReferencePublication {
      get { return "Anthony Bagnall, Jason Lines, William Vickers and Eamonn Keogh, The UEA & UCR Time Series Classification Repository, www.timeseriesclassification.com"; }
    }

    public override IClassificationProblemData LoadData(IDataDescriptor id) {
      var descriptor = (TimeSeriesDataDescriptor)id;
      using (var instancesZipFile = OpenZipArchive()) {
        var trainingEntry = instancesZipFile.GetEntry(descriptor.TrainingEntryName);
        var testEntry = instancesZipFile.GetEntry(descriptor.TestEntryName);

        if (trainingEntry == null || testEntry == null) {
          throw new InvalidOperationException("The training or test entry could not be found in the archive.");
        }

        using (var trainingReader = new StreamReader(trainingEntry.Open()))
        using (var testReader = new StreamReader(testEntry.Open())) {
          ParseMetadata(trainingReader, out var inputVariables, out string targetVariable, out var classLabels);
          ParseMetadata(testReader, out _, out _, out _); // ignore outputs

          // Read data
          var inputsData = new List<DoubleVector>[inputVariables.Count];
          for (int i = 0; i < inputsData.Length; i++) inputsData[i] = new List<DoubleVector>();
          bool numericTarget = classLabels.All(label => !double.IsNaN(ParseNumber(label)));
          IList targetData = numericTarget ? new List<double>() : new List<string>() as IList;
          ReadData(trainingReader, inputsData, targetData, out int numTrainingRows);
          ReadData(testReader, inputsData, targetData, out int numTestRows);

          // Translate class values to numeric values
          if (targetData is List<string> stringTargetData) {
            var labelTranslation = classLabels
              .Select((x, i) => new { Label = x, i })
              .ToDictionary(x => x.Label, x => (double)x.i);
            targetData = stringTargetData.Select(label => labelTranslation[label]).ToList();
          }

          // Build dataset
          var dataset = new Dataset(
            inputVariables.Concat(new[] { targetVariable }),
            inputsData.Concat(new[] { targetData })
          );
          Debug.Assert(dataset.Rows == numTrainingRows + numTestRows);
          Debug.Assert(dataset.Columns == inputVariables.Count + 1);

          // Build problem data
          var problemData = new ClassificationProblemData(dataset, inputVariables, targetVariable) {
            Name = descriptor.Name
          };
          problemData.TrainingPartition.Start = 0;
          problemData.TrainingPartition.End = numTrainingRows;
          problemData.TestPartition.Start = numTrainingRows;
          problemData.TestPartition.End = numTrainingRows + numTestRows;

          return problemData;
        }
      }
    }

    private static void ParseMetadata(StreamReader reader, out List<string> inputVariables, out string targetVariable, out List<string> classLabels) {
      int nrOfInputs = 0;
      IEnumerable<string> labels = null;
      bool dataStart = false;

      while (!reader.EndOfStream && !dataStart) {
        var line = reader.ReadLine();
        if (line.StartsWith("#")) {
          // Comment
        } else if (line.StartsWith("@")) {
          var splits = line.Split(' ');
          var type = splits.First();
          var arguments = splits.Skip(1).ToList();
          switch (type) {
            case "@univariate":
              bool univariate = bool.Parse(arguments[0]);
              if (univariate)
                nrOfInputs = 1;
              break;
            case "@dimensions":
              int dimensions = int.Parse(arguments[0]);
              nrOfInputs = dimensions;
              break;
            case "@classLabel":
              bool containLabels = bool.Parse(arguments[0]);
              if (containLabels)
                labels = arguments.Skip(1);
              break;
            case "@data":
              dataStart = true;
              break;
          }
        } else {
          throw new InvalidOperationException("A data section already occurred within metadata section.");
        }
      }

      int digits = Math.Max((int)Math.Log10(nrOfInputs - 1) + 1, 1);
      inputVariables = Enumerable.Range(0, nrOfInputs)
        .Select(i => "X" + i.ToString("D" + digits))
        .ToList();

      targetVariable = "Y";

      classLabels = labels.ToList();
    }

    private static void ReadData(StreamReader reader, List<DoubleVector>[] inputsData, IList targetData, out int count) {
      var numericTargetData = targetData as List<double>;
      var stringTargetData = targetData as List<string>;

      count = 0;
      while (!reader.EndOfStream) {
        var line = reader.ReadLine();
        var variables = line.Split(':');

        // parse all except last, which is the non-vector target
        for (int i = 0; i < variables.Length - 1; i++) {
          var variable = variables[i];
          var numbers = variable
            .Split(',')
            .Select(ParseNumber)
            .ToList();
          inputsData[i].Add(new DoubleVector(numbers));
        }

        var target = variables[variables.Length - 1];
        if (numericTargetData != null) numericTargetData.Add(ParseNumber(target));
        else if (stringTargetData != null) stringTargetData.Add(target);
        else throw new InvalidOperationException("Target must either be numeric or a string.");

        count++;
      }
    }

    private static double ParseNumber(string number) {
      return
        double.TryParse(number, NumberStyles.Float, CultureInfo.InvariantCulture, out double parsed)
          ? parsed
          : double.NaN;
    }

    public override IEnumerable<IDataDescriptor> GetDataDescriptors() {
      using (var instancesZipFile = OpenZipArchive()) {
        var instances = GroupEntriesByInstance(instancesZipFile.Entries);
        var descriptors = instances.Select(instance => CreateDescriptor(instance.Key, instance.Value));

        return descriptors.ToList();
      }
    }

    private ZipArchive OpenZipArchive() {
      var instanceArchiveName = Path.Combine("Classification", "Data", FileName + ".zip");
      var stream = new FileStream(instanceArchiveName, FileMode.Open, FileAccess.Read, FileShare.Read);
      return new ZipArchive(stream, ZipArchiveMode.Read);
    }

    private static IDictionary<string, List<ZipArchiveEntry>> GroupEntriesByInstance(ReadOnlyCollection<ZipArchiveEntry> entries) {
      var topLevelEntries = entries.Where(entry => string.IsNullOrEmpty(entry.Name)).ToList();

      return topLevelEntries.ToDictionary(
        entry => Path.GetDirectoryName(entry.FullName),
        entry => entries.Except(topLevelEntries).Where(subEntry => subEntry.FullName.StartsWith(entry.FullName)).ToList());
    }

    private static TimeSeriesDataDescriptor CreateDescriptor(string name, List<ZipArchiveEntry> subEntries) {
      var trainingEntry = subEntries.Single(entry => entry.Name.EndsWith("_TRAIN.ts"));
      var testEntry = subEntries.Single(entry => entry.Name.EndsWith("_TEST.ts"));
      return new TimeSeriesDataDescriptor(name, trainingEntry.FullName, testEntry.FullName);
    }
  }
}