Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
07/07/14 15:33:56 (10 years ago)
Author:
mkommend
Message:

#2206: Merged data preprocessing into the trunk.

Location:
trunk/sources
Files:
18 edited
4 copied

Legend:

Unmodified
Added
Removed

  • trunk/sources

  • trunk/sources/HeuristicLab.Problems.DataAnalysis

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Dataset.cs

    r10406 r11114  
    170170        yield return value;
    171171    }
     172
     173    public IEnumerable<string> GetStringValues(string variableName) {
     174      IList list;
     175      if (!variableValues.TryGetValue(variableName, out list))
     176        throw new ArgumentException("The variable " + variableName + " does not exist in the dataset.");
     177      List<string> values = list as List<string>;
     178      if (values == null) throw new ArgumentException("The variable " + variableName + " is not a string variable.");
     179
     180      //mkommend yield return used to enable lazy evaluation
     181      foreach (string value in values)
     182        yield return value;
     183    }
     184
     185    public IEnumerable<DateTime> GetDateTimeValues(string variableName) {
     186      IList list;
     187      if (!variableValues.TryGetValue(variableName, out list))
     188        throw new ArgumentException("The variable " + variableName + " does not exist in the dataset.");
     189      List<DateTime> values = list as List<DateTime>;
     190      if (values == null) throw new ArgumentException("The variable " + variableName + " is not a datetime variable.");
     191
     192      //mkommend yield return used to enable lazy evaluation
     193      foreach (DateTime value in values)
     194        yield return value;
     195    }
     196
    172197    public ReadOnlyCollection<double> GetReadOnlyDoubleValues(string variableName) {
    173198      IList list;
    … …  
    194219
    195220      return rows.Select(index => values[index]);
     221    }
     222
     223    public bool IsType<T>(string variableName) {
     224      return variableValues[variableName] is IList<T>;
    196225    }
    197226

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj

    r9973 r11114  
    141141    <Compile Include="Implementation\TimeSeriesPrognosis\TimeSeriesPrognosisSolution.cs" />
    142142    <Compile Include="Implementation\TimeSeriesPrognosis\TimeSeriesPrognosisSolutionBase.cs" />
     143    <Compile Include="Implementation\Transformations\CopyColumnTransformation.cs" />
     144    <Compile Include="Implementation\Transformations\ExponentialTransformation.cs" />
     145    <Compile Include="Implementation\Transformations\LinearTransformation.cs" />
     146    <Compile Include="Implementation\Transformations\LogarithmicTransformation.cs" />
     147    <Compile Include="Implementation\Transformations\PowerTransformation.cs" />
     148    <Compile Include="Implementation\Transformations\ReciprocalTransformation.cs" />
     149    <Compile Include="Implementation\Transformations\ShiftStandardDistributionTransformation.cs" />
     150    <Compile Include="Implementation\Transformations\ShiftToRangeTransformation.cs" />
     151    <Compile Include="Implementation\Transformations\Transformation.cs" />
    143152    <Compile Include="Interfaces\Classification\IClassificationEnsembleModel.cs">
    144153      <SubType>Code</SubType>
    … …  
    150159    <Compile Include="Interfaces\IDataAnalysisSolutionExporter.cs" />
    151160    <Compile Include="Interfaces\IDependencyCalculator.cs" />
     161    <Compile Include="Interfaces\IModelBacktransformator.cs" />
     162    <Compile Include="Interfaces\ITransformation.cs" />
     163    <Compile Include="Interfaces\ITransformationMapper.cs" />
    152164    <Compile Include="Interfaces\Regression\IRegressionEnsembleModel.cs">
    153165      <SubType>Code</SubType>

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs

    r10540 r11114  
    300300    }
    301301
    302     public ClassificationProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, string targetVariable)
    303       : base(dataset, allowedInputVariables) {
     302    public ClassificationProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, string targetVariable, IEnumerable<ITransformation> transformations = null)
     303      : base(dataset, allowedInputVariables, transformations ?? Enumerable.Empty<ITransformation>()) {
    304304      var validTargetVariableValues = CheckVariablesForPossibleTargetVariables(dataset).Select(x => new StringValue(x).AsReadOnly()).ToList();
    305305      var target = validTargetVariableValues.Where(x => x.Value == targetVariable).DefaultIfEmpty(validTargetVariableValues.First()).First();

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringProblemData.cs

    r9456 r11114  
    2121
    2222using System.Collections.Generic;
     23using System.Linq;
    2324using HeuristicLab.Common;
    2425using HeuristicLab.Core;
    … …  
    8687    }
    8788
    88     public ClusteringProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables)
    89       : base(dataset, allowedInputVariables) {
     89    public ClusteringProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, IEnumerable<ITransformation> transformations = null)
     90      : base(dataset, allowedInputVariables, transformations ?? Enumerable.Empty<ITransformation>()) {
    9091    }
    9192  }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblemData.cs

    r10540 r11114  
    3838    protected const string TrainingPartitionParameterName = "TrainingPartition";
    3939    protected const string TestPartitionParameterName = "TestPartition";
     40    protected const string TransformationsParameterName = "Transformations";
    4041
    4142    #region parameter properites
    … …  
    5152    public IFixedValueParameter<IntRange> TestPartitionParameter {
    5253      get { return (IFixedValueParameter<IntRange>)Parameters[TestPartitionParameterName]; }
     54    }
     55    public IFixedValueParameter<ReadOnlyItemList<ITransformation>> TransformationsParameter {
     56      get { return (IFixedValueParameter<ReadOnlyItemList<ITransformation>>)Parameters[TransformationsParameterName]; }
    5357    }
    5458    #endregion
    … …  
    8993    }
    9094
     95    public IEnumerable<ITransformation> Transformations {
     96      get { return TransformationsParameter.Value; }
     97    }
     98
    9199    public virtual bool IsTrainingSample(int index) {
    92100      return index >= 0 && index < Dataset.Rows &&
    … …  
    111119    [StorableHook(HookType.AfterDeserialization)]
    112120    private void AfterDeserialization() {
     121      if (!Parameters.ContainsKey(TransformationsParameterName)) {
     122        Parameters.Add(new FixedValueParameter<ReadOnlyItemList<ITransformation>>(TransformationsParameterName, "", new ItemList<ITransformation>().AsReadOnly()));
     123        TransformationsParameter.Hidden = true;
     124      }
    113125      RegisterEventHandlers();
    114126    }
    115127
    116     protected DataAnalysisProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables) {
     128    protected DataAnalysisProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, IEnumerable<ITransformation> transformations) {
    117129      if (dataset == null) throw new ArgumentNullException("The dataset must not be null.");
    118130      if (allowedInputVariables == null) throw new ArgumentNullException("The allowedInputVariables must not be null.");
    … …  
    120132      if (allowedInputVariables.Except(dataset.DoubleVariables).Any())
    121133        throw new ArgumentException("All allowed input variables must be present in the dataset and of type double.");
     134
     135      if (transformations == null) throw new ArgumentNullException("The transformations must not be null.");
    122136
    123137      var inputVariables = new CheckedItemList<StringValue>(dataset.DoubleVariables.Select(x => new StringValue(x)));
    … …  
    130144      int testPartitionEnd = dataset.Rows;
    131145
     146      var transformationsList = new ItemList<ITransformation>(transformations);
     147
    132148      Parameters.Add(new FixedValueParameter<Dataset>(DatasetParameterName, "", dataset));
    133149      Parameters.Add(new FixedValueParameter<ReadOnlyCheckedItemList<StringValue>>(InputVariablesParameterName, "", inputVariables.AsReadOnly()));
    134150      Parameters.Add(new FixedValueParameter<IntRange>(TrainingPartitionParameterName, "", new IntRange(trainingPartitionStart, trainingPartitionEnd)));
    135151      Parameters.Add(new FixedValueParameter<IntRange>(TestPartitionParameterName, "", new IntRange(testPartitionStart, testPartitionEnd)));
     152      Parameters.Add(new FixedValueParameter<ReadOnlyItemList<ITransformation>>(TransformationsParameterName, "", transformationsList.AsReadOnly()));
     153
     154      TransformationsParameter.Hidden = true;
    136155
    137156      ((ValueParameter<Dataset>)DatasetParameter).ReactOnValueToStringChangedAndValueItemImageChanged = false;
    … …  
    144163      TrainingPartition.ValueChanged += new EventHandler(Parameter_ValueChanged);
    145164      TestPartition.ValueChanged += new EventHandler(Parameter_ValueChanged);
     165      TransformationsParameter.ValueChanged += new EventHandler(Parameter_ValueChanged);
    146166    }
    147167

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblemData.cs

    r10540 r11114  
    137137    }
    138138
    139     public RegressionProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, string targetVariable)
    140       : base(dataset, allowedInputVariables) {
     139    public RegressionProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, string targetVariable, IEnumerable<ITransformation> transformations = null)
     140      : base(dataset, allowedInputVariables, transformations ?? Enumerable.Empty<ITransformation>()) {
    141141      var variables = InputVariables.Select(x => x.AsReadOnly()).ToList();
    142142      Parameters.Add(new ConstrainedValueParameter<StringValue>(TargetVariableParameterName, new ItemSet<StringValue>(variables), variables.Where(x => x.Value == targetVariable).First()));

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisProblemData.cs

    r11031 r11114  
    15821582      TrainingPartition.Start = 50;
    15831583    }
    1584     public TimeSeriesPrognosisProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, string targetVariable)
    1585       : base(dataset, allowedInputVariables, targetVariable) {
     1584    public TimeSeriesPrognosisProblemData(Dataset dataset, IEnumerable<string> allowedInputVariables, string targetVariable, IEnumerable<ITransformation> transformations = null)
     1585      : base(dataset, allowedInputVariables, targetVariable, transformations ?? Enumerable.Empty<ITransformation>()) {
    15861586      Parameters.Add(new FixedValueParameter<IntValue>(TrainingHorizonParameterName, "Specifies the horizon (how far the prognosis reaches in the future) for each training sample.", new IntValue(1)));
    15871587      Parameters.Add(new FixedValueParameter<IntValue>(TestHorizonParameterName, "Specifies the horizon (how far the prognosis reaches in the future) for each test sample.", new IntValue(1)));
    … …  
    16431643
    16441644      var trainingDataStart = TrainingIndices.First();
    1645      
     1645
    16461646      base.AdjustProblemDataProperties(problemData);
    1647      
     1647
    16481648      TestPartition.Start = trainingDataStart;
    16491649

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Transformations/ExponentialTransformation.cs

    r11068 r11114  
    2929    [StorableConstructor]
    3030    protected ExponentialTransformation(bool deserializing) : base(deserializing) { }
    31     protected ExponentialTransformation(Transformation<double> original, Cloner cloner)
     31
     32    protected ExponentialTransformation(ExponentialTransformation original, Cloner cloner)
    3233      : base(original, cloner) {
    3334    }
     35
    3436    public ExponentialTransformation(IEnumerable<string> allowedColumns)
    3537      : base(allowedColumns) {

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Transformations/LinearTransformation.cs

    r11068 r11114  
    6666    [StorableConstructor]
    6767    protected LinearTransformation(bool deserializing) : base(deserializing) { }
    68     protected LinearTransformation(Transformation<double> original, Cloner cloner)
     68    protected LinearTransformation(LinearTransformation original, Cloner cloner)
    6969      : base(original, cloner) {
    7070    }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Transformations/LogarithmicTransformation.cs

    r11068 r11114  
    3030    [StorableConstructor]
    3131    protected LogarithmicTransformation(bool deserializing) : base(deserializing) { }
    32     protected LogarithmicTransformation(Transformation<double> original, Cloner cloner)
     32    protected LogarithmicTransformation(LogarithmicTransformation original, Cloner cloner)
    3333      : base(original, cloner) {
    3434    }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Transformations/PowerTransformation.cs

    r11068 r11114  
    2929    [StorableConstructor]
    3030    protected PowerTransformation(bool deserializing) : base(deserializing) { }
    31     protected PowerTransformation(Transformation<double> original, Cloner cloner)
     31    protected PowerTransformation(PowerTransformation original, Cloner cloner)
    3232      : base(original, cloner) {
    3333    }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Transformations/ReciprocalTransformation.cs

    r11068 r11114  
    1919    [StorableConstructor]
    2020    protected ReciprocalTransformation(bool deserializing) : base(deserializing) { }
    21     protected ReciprocalTransformation(Transformation<double> original, Cloner cloner)
     21    protected ReciprocalTransformation(ReciprocalTransformation original, Cloner cloner)
    2222      : base(original, cloner) {
    2323    }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Transformations/ShiftStandardDistributionTransformation.cs

    r11068 r11114  
    5252    [StorableConstructor]
    5353    protected ShiftStandardDistributionTransformation(bool deserializing) : base(deserializing) { }
    54     protected ShiftStandardDistributionTransformation(Transformation<double> original, Cloner cloner)
     54    protected ShiftStandardDistributionTransformation(ShiftStandardDistributionTransformation original, Cloner cloner)
    5555      : base(original, cloner) {
    5656    }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Transformations/ShiftToRangeTransformation.cs

    r11068 r11114  
    2929    [StorableConstructor]
    3030    protected ShiftToRangeTransformation(bool deserializing) : base(deserializing) { }
    31     protected ShiftToRangeTransformation(Transformation<double> original, Cloner cloner)
     31    protected ShiftToRangeTransformation(ShiftToRangeTransformation original, Cloner cloner)
    3232      : base(original, cloner) {
    3333    }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/IDataAnalysisProblemData.cs

    r10540 r11114  
    3939    IEnumerable<int> TestIndices { get; }
    4040
     41    IEnumerable<ITransformation> Transformations { get; }
     42
    4143    bool IsTrainingSample(int index);
    4244    bool IsTestSample(int index);

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Plugin.cs.frame

    r10037 r11114  
    3737  [PluginDependency("HeuristicLab.Parameters", "3.3")]
    3838  [PluginDependency("HeuristicLab.Persistence", "3.3")]
    39   [PluginDependency("HeuristicLab.Problems.Instances", "3.3")]
     39  [PluginDependency("HeuristicLab.Problems.Instances", "3.3")] 
    4040  public class HeuristicLabProblemsDataAnalysisPlugin : PluginBase {
    4141  }
Note: See TracChangeset for help on using the changeset viewer.