Free cookie consent management tool by TermsFeed Policy Generator

source: branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblemData.cs @ 15879

Last change on this file since 15879 was 15879, checked in by pfleck, 6 years ago

#2906 minor refactoring

File size: 14.1 KB
RevLine 
[5540]1#region License Information
2/* HeuristicLab
[15583]3 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[5540]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
[10540]25using System.Text;
[5586]26using HeuristicLab.Collections;
[5540]27using HeuristicLab.Common;
28using HeuristicLab.Core;
[5586]29using HeuristicLab.Data;
30using HeuristicLab.Parameters;
[5540]31using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
32
33namespace HeuristicLab.Problems.DataAnalysis {
34  [StorableClass]
[5586]35  public abstract class DataAnalysisProblemData : ParameterizedNamedItem, IDataAnalysisProblemData {
[6666]36    protected const string DatasetParameterName = "Dataset";
37    protected const string InputVariablesParameterName = "InputVariables";
38    protected const string TrainingPartitionParameterName = "TrainingPartition";
39    protected const string TestPartitionParameterName = "TestPartition";
[11114]40    protected const string TransformationsParameterName = "Transformations";
[5586]41
42    #region parameter properites
[14507]43    //mkommend: inserted parameter caching due to performance reasons
44    private IFixedValueParameter<Dataset> datasetParameter;
[5601]45    public IFixedValueParameter<Dataset> DatasetParameter {
[14507]46      get {
47        if (datasetParameter == null) datasetParameter = (IFixedValueParameter<Dataset>)Parameters[DatasetParameterName];
48        return datasetParameter;
49      }
[5586]50    }
[14507]51
52    private IFixedValueParameter<ReadOnlyCheckedItemList<StringValue>> inputVariablesParameter;
[5847]53    public IFixedValueParameter<ReadOnlyCheckedItemList<StringValue>> InputVariablesParameter {
[14507]54      get {
55        if (inputVariablesParameter == null) inputVariablesParameter = (IFixedValueParameter<ReadOnlyCheckedItemList<StringValue>>)Parameters[InputVariablesParameterName];
56        return inputVariablesParameter;
57      }
[5586]58    }
[14507]59
60    private IFixedValueParameter<IntRange> trainingPartitionParameter;
[5759]61    public IFixedValueParameter<IntRange> TrainingPartitionParameter {
[14507]62      get {
63        if (trainingPartitionParameter == null) trainingPartitionParameter = (IFixedValueParameter<IntRange>)Parameters[TrainingPartitionParameterName];
64        return trainingPartitionParameter;
65      }
[5586]66    }
[14507]67
68    private IFixedValueParameter<IntRange> testPartitionParameter;
[5759]69    public IFixedValueParameter<IntRange> TestPartitionParameter {
[14507]70      get {
71        if (testPartitionParameter == null) testPartitionParameter = (IFixedValueParameter<IntRange>)Parameters[TestPartitionParameterName];
72        return testPartitionParameter;
73      }
[5586]74    }
[14507]75
[15846]76    public IFixedValueParameter<ReadOnlyItemList<IDataAnalysisTransformation>> TransformationsParameter {
77      get { return (IFixedValueParameter<ReadOnlyItemList<IDataAnalysisTransformation>>)Parameters[TransformationsParameterName]; }
[11114]78    }
[5586]79    #endregion
80
[6666]81    #region properties
82    protected bool isEmpty = false;
83    public bool IsEmpty {
84      get { return isEmpty; }
85    }
[12509]86    public IDataset Dataset {
[5586]87      get { return DatasetParameter.Value; }
[5540]88    }
[5649]89    public ICheckedItemList<StringValue> InputVariables {
[5586]90      get { return InputVariablesParameter.Value; }
91    }
[5554]92    public IEnumerable<string> AllowedInputVariables {
[5649]93      get { return InputVariables.CheckedItems.Select(x => x.Value.Value); }
[5540]94    }
95
[14843]96    public double[,] AllowedInputsTrainingValues {
97      get { return Dataset.ToArray(AllowedInputVariables, TrainingIndices); }
98    }
99
100    public double[,] AllowedInputsTestValues { get { return Dataset.ToArray(AllowedInputVariables, TestIndices); } }
[5759]101    public IntRange TrainingPartition {
102      get { return TrainingPartitionParameter.Value; }
[5540]103    }
[5759]104    public IntRange TestPartition {
105      get { return TestPartitionParameter.Value; }
[5540]106    }
[5554]107
[13766]108    public virtual IEnumerable<int> AllIndices {
109      get { return Enumerable.Range(0, Dataset.Rows); }
110    }
[8139]111    public virtual IEnumerable<int> TrainingIndices {
[5554]112      get {
[7265]113        return Enumerable.Range(TrainingPartition.Start, Math.Max(0, TrainingPartition.End - TrainingPartition.Start))
[6672]114                         .Where(IsTrainingSample);
[5540]115      }
116    }
[8139]117    public virtual IEnumerable<int> TestIndices {
[5554]118      get {
[7265]119        return Enumerable.Range(TestPartition.Start, Math.Max(0, TestPartition.End - TestPartition.Start))
[6672]120           .Where(IsTestSample);
[5554]121      }
122    }
[6672]123
[15846]124    public IEnumerable<IDataAnalysisTransformation> Transformations {
[11114]125      get { return TransformationsParameter.Value; }
126    }
127
[6672]128    public virtual bool IsTrainingSample(int index) {
129      return index >= 0 && index < Dataset.Rows &&
[14507]130             TrainingPartition.Start <= index && index < TrainingPartition.End &&
131             (index < TestPartition.Start || TestPartition.End <= index);
[6672]132    }
133
134    public virtual bool IsTestSample(int index) {
135      return index >= 0 && index < Dataset.Rows &&
136             TestPartition.Start <= index && index < TestPartition.End;
137    }
[5540]138    #endregion
139
[6236]140    protected DataAnalysisProblemData(DataAnalysisProblemData original, Cloner cloner)
141      : base(original, cloner) {
[6666]142      isEmpty = original.isEmpty;
[6236]143      RegisterEventHandlers();
144    }
[5540]145    [StorableConstructor]
146    protected DataAnalysisProblemData(bool deserializing) : base(deserializing) { }
[8542]147
[6581]148    [StorableHook(HookType.AfterDeserialization)]
149    private void AfterDeserialization() {
[15846]150      if (Parameters[TransformationsParameterName] is FixedValueParameter<ReadOnlyItemList<ITransformation>>)
151        Parameters.Remove(TransformationsParameterName);
152      if (!Parameters.ContainsKey(TransformationsParameterName))
153        Parameters.Add(new FixedValueParameter<ReadOnlyItemList<IDataAnalysisTransformation>>(TransformationsParameterName, new ItemList<IDataAnalysisTransformation>().AsReadOnly()) { Hidden = true });
154
[6581]155      RegisterEventHandlers();
156    }
[5559]157
[15846]158    protected DataAnalysisProblemData(IDataset dataset, IEnumerable<string> allowedInputVariables, IEnumerable<IDataAnalysisTransformation> transformations = null) {
[5559]159      if (dataset == null) throw new ArgumentNullException("The dataset must not be null.");
[14826]160      if (allowedInputVariables == null) throw new ArgumentNullException("The allowed input variables must not be null.");
[5559]161
[14826]162      if (allowedInputVariables.Except(dataset.DoubleVariables).Except(dataset.StringVariables).Any())
163        throw new ArgumentException("All allowed input variables must be present in the dataset and of type double or string.");
[5554]164
[14826]165      var variables = dataset.VariableNames.Where(variable => dataset.VariableHasType<double>(variable) || dataset.VariableHasType<string>(variable));
166      var inputVariables = new CheckedItemList<StringValue>(variables.Select(x => new StringValue(x)));
[5586]167      foreach (StringValue x in inputVariables)
168        inputVariables.SetItemCheckedState(x, allowedInputVariables.Contains(x.Value));
[5540]169
[5586]170      int trainingPartitionStart = 0;
171      int trainingPartitionEnd = dataset.Rows / 2;
172      int testPartitionStart = dataset.Rows / 2;
173      int testPartitionEnd = dataset.Rows;
[5540]174
[15846]175      var transformationsList = new ItemList<IDataAnalysisTransformation>(transformations ?? Enumerable.Empty<IDataAnalysisTransformation>());
[11114]176
[12509]177      Parameters.Add(new FixedValueParameter<Dataset>(DatasetParameterName, "", (Dataset)dataset));
[5847]178      Parameters.Add(new FixedValueParameter<ReadOnlyCheckedItemList<StringValue>>(InputVariablesParameterName, "", inputVariables.AsReadOnly()));
[5759]179      Parameters.Add(new FixedValueParameter<IntRange>(TrainingPartitionParameterName, "", new IntRange(trainingPartitionStart, trainingPartitionEnd)));
180      Parameters.Add(new FixedValueParameter<IntRange>(TestPartitionParameterName, "", new IntRange(testPartitionStart, testPartitionEnd)));
[15846]181      Parameters.Add(new FixedValueParameter<ReadOnlyItemList<IDataAnalysisTransformation>>(TransformationsParameterName, "", transformationsList.AsReadOnly()) { Hidden = transformationsList.Count == 0 });
[5586]182
[5601]183      ((ValueParameter<Dataset>)DatasetParameter).ReactOnValueToStringChangedAndValueItemImageChanged = false;
[5586]184      RegisterEventHandlers();
[5540]185    }
[5542]186
[5586]187    private void RegisterEventHandlers() {
188      DatasetParameter.ValueChanged += new EventHandler(Parameter_ValueChanged);
[5649]189      InputVariables.CheckedItemsChanged += new CollectionItemsChangedEventHandler<IndexedItem<StringValue>>(InputVariables_CheckedItemsChanged);
[5759]190      TrainingPartition.ValueChanged += new EventHandler(Parameter_ValueChanged);
191      TestPartition.ValueChanged += new EventHandler(Parameter_ValueChanged);
[11114]192      TransformationsParameter.ValueChanged += new EventHandler(Parameter_ValueChanged);
[5540]193    }
194
[5649]195    private void InputVariables_CheckedItemsChanged(object sender, CollectionItemsChangedEventArgs<IndexedItem<StringValue>> e) {
[5586]196      OnChanged();
197    }
[5649]198
[5586]199    private void Parameter_ValueChanged(object sender, EventArgs e) {
200      OnChanged();
201    }
202
[5554]203    public event EventHandler Changed;
204    protected virtual void OnChanged() {
[5540]205      var listeners = Changed;
206      if (listeners != null) listeners(this, EventArgs.Empty);
207    }
[10540]208
209    protected virtual bool IsProblemDataCompatible(IDataAnalysisProblemData problemData, out string errorMessage) {
210      errorMessage = string.Empty;
211      if (problemData == null) throw new ArgumentNullException("problemData", "The provided problemData is null.");
212
213      //check allowed input variables
214      StringBuilder message = new StringBuilder();
215      var variables = new HashSet<string>(problemData.InputVariables.Select(x => x.Value));
216      foreach (var item in AllowedInputVariables) {
217        if (!variables.Contains(item))
218          message.AppendLine("Input variable '" + item + "' is not present in the new problem data.");
219      }
220
221      if (message.Length != 0) {
222        errorMessage = message.ToString();
223        return false;
224      }
225      return true;
226
227    }
228
229    public virtual void AdjustProblemDataProperties(IDataAnalysisProblemData problemData) {
230      DataAnalysisProblemData data = problemData as DataAnalysisProblemData;
231      if (data == null) throw new ArgumentException("The problem data is not a data analysis problem data. Instead a " + problemData.GetType().GetPrettyName() + " was provided.", "problemData");
232
233      string errorMessage;
234      if (!data.IsProblemDataCompatible(this, out errorMessage)) {
235        throw new InvalidOperationException(errorMessage);
236      }
237
238      foreach (var inputVariable in InputVariables) {
239        var variable = data.InputVariables.FirstOrDefault(i => i.Value == inputVariable.Value);
240        InputVariables.SetItemCheckedState(inputVariable, variable != null && data.InputVariables.ItemChecked(variable));
241      }
242    }
[15847]243
[15856]244    public virtual IDataAnalysisProblemData Transform() {
[15870]245      var newDataset = DataAnalysisTransformationModel.Transform(Dataset, Transformations);
[15856]246
[15879]247      var extendedInputs = DataAnalysisTransformationModel.ExtendInputVariables(AllowedInputVariables, Transformations);
[15856]248      var checkedInputs = new CheckedItemList<StringValue>(newDataset.VariableNames.Select(x => new StringValue(x)));
249      foreach (var input in checkedInputs) checkedInputs.SetItemCheckedState(input, extendedInputs.Contains(input.Value));
250
251      // TODO: Cannot create concrete instance here (maybe derived Create-method?)
252      var cloner = new Cloner();
253      cloner.RegisterClonedObject(Dataset, newDataset);
254      cloner.RegisterClonedObject(InputVariables, checkedInputs.AsReadOnly());
[15870]255      // TODO: valid values for target are not extended
[15856]256
257      return cloner.Clone(this);
258    }
259
[15847]260    public virtual IDataAnalysisProblemData InverseTransform() {
261      var newDataset = InverseTransform(Dataset, Transformations);
262
[15856]263      var checkedInputs = new CheckedItemList<StringValue>(newDataset.VariableNames.Select(x => new StringValue(x)));
264      foreach (var input in checkedInputs) checkedInputs.SetItemCheckedState(input, AllowedInputVariables.Contains(input.Value));
265
[15847]266      // TODO: Cannot create concrete instance here (maybe derived Create-method?)
267      var cloner = new Cloner();
268      cloner.RegisterClonedObject(Dataset, newDataset);
[15856]269      cloner.RegisterClonedObject(InputVariables, checkedInputs.AsReadOnly());
[15870]270      // TODO: check valid target values
[15847]271
272      return cloner.Clone(this);
273    }
274
[15848]275    public static IDataset InverseTransform(IDataset dataset, IEnumerable<IDataAnalysisTransformation> transformations, bool removeVirtualVariables = true) {
[15847]276      var modifiableDataset = ((Dataset)dataset).ToModifiable();
277
[15848]278      var transformationsStack = new Stack<IDataAnalysisTransformation>(transformations);
279      while (transformationsStack.Any()) {
280        var transformation = transformationsStack.Pop();
[15847]281        var trans = (ITransformation<double>)transformation.Transformation;
282
[15856]283        var prevTransformations = transformations.Except(transformationsStack);
284        bool originalWasChanged = prevTransformations.Any(x => x.TransformedVariable == transformation.OriginalVariable);
285        if (originalWasChanged) {
286          var transformedData = modifiableDataset.GetDoubleValues(transformation.TransformedVariable);
[15847]287
[15856]288          var originalData = trans.InverseApply(transformedData).ToList();
289          modifiableDataset.ReplaceVariable(transformation.OriginalVariable, originalData);
290        }
[15848]291
[15858]292        bool transformedVariablePending = transformationsStack.Any(x => x.OriginalVariable == transformation.TransformedVariable || x.TransformedVariable == transformation.TransformedVariable);
[15848]293        if (removeVirtualVariables && !transformedVariablePending)
294          modifiableDataset.RemoveVariable(transformation.TransformedVariable);
[15847]295      }
296
297      return modifiableDataset;
298    }
[5540]299  }
300}
Note: See TracBrowser for help on using the repository browser.