Changeset 15870 for branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4

Timestamp:

03/28/18 17:17:31 (6 years ago)

Author:

pfleck

Message:

#2906

• Implemented for classification, clustering, etc.
• Simplified Transformation interfaces (read-only, ...).
• Started moving transformation logic out of ProblemData.

Location:

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4

Files:

• HeuristicLab.Problems.DataAnalysis-3.4.csproj (modified) (5 diffs)
• Implementation/Classification/ClassificationTransformationModel.cs (added)
• Implementation/Clustering/ClusteringTransformationModel.cs (added)
• Implementation/DataAnalysisProblemData.cs (modified) (3 diffs)
• Implementation/DataAnalysisTransformation.cs (modified) (4 diffs)
• Implementation/DataAnalysisTransformationModel.cs (moved) (moved from branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/TransformedRegressionModel.cs) (3 diffs)
• Implementation/Regression/RegressionProblemData.cs (modified) (2 diffs)
• Implementation/Regression/RegressionTransformationModel.cs (added)
• Implementation/Regression/TransformedRegressionSolution.cs (deleted)
• Implementation/TimeSeriesPrognosis/Models/TimeSeriesPrognosisTransformationModel.cs (added)
• Interfaces/Classification/IClassificationTransformationModel.cs (added)
• Interfaces/Clustering/IClusteringTransformationModel.cs (added)
• Interfaces/IDataAnalysisTransformation.cs (modified) (1 diff)
• Interfaces/IDataAnalysisTransformationModel.cs (added)
• Interfaces/Regression/IRegressionTransformationModel.cs (moved) (moved from branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/Regression/ITransformedRegressionModel.cs) (1 diff)
• Interfaces/Regression/ITransformedRegressionSolution.cs (deleted)
• Interfaces/TimeSeriesPrognosis/ITimeSeriesPrognosisTransformationModel.cs (added)

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj

@@ -187 +187 @@
     <Compile Include="Implementation\Classification\ClassificationSolutionBase.cs" />
     <Compile Include="Implementation\Classification\ClassificationSolutionVariableImpactsCalculator.cs" />
+    <Compile Include="Implementation\Classification\ClassificationTransformationModel.cs" />
     <Compile Include="Implementation\Classification\ConstantClassificationSolution.cs" />
     <Compile Include="Implementation\Classification\DiscriminantFunctionClassificationSolutionBase.cs" />
@@ -193 +194 @@
     <Compile Include="Implementation\Clustering\ClusteringProblemData.cs" />
     <Compile Include="Implementation\Clustering\ClusteringSolution.cs" />
+    <Compile Include="Implementation\Clustering\ClusteringTransformationModel.cs" />
     <Compile Include="Implementation\ConstantModel.cs" />
     <Compile Include="Implementation\DataAnalysisModel.cs" />
@@ -206 +208 @@
     <Compile Include="Implementation\Regression\RegressionModel.cs" />
     <Compile Include="Implementation\Regression\RegressionSolutionVariableImpactsCalculator.cs" />
-    <Compile Include="Implementation\Regression\TransformedRegressionModel.cs" />
-    <Compile Include="Implementation\Regression\TransformedRegressionSolution.cs" />
+    <Compile Include="Implementation\DataAnalysisTransformationModel.cs" />
+    <Compile Include="Implementation\Regression\RegressionTransformationModel.cs" />
     <Compile Include="Implementation\TimeSeriesPrognosis\Models\ConstantTimeSeriesPrognosisModel.cs" />
     <Compile Include="Implementation\TimeSeriesPrognosis\Models\TimeSeriesPrognosisAutoRegressiveModel.cs" />
+    <Compile Include="Implementation\TimeSeriesPrognosis\Models\TimeSeriesPrognosisTransformationModel.cs" />
     <Compile Include="Implementation\TimeSeriesPrognosis\TimeSeriesPrognosisProblem.cs" />
     <Compile Include="Implementation\TimeSeriesPrognosis\TimeSeriesPrognosisProblemData.cs" />
@@ -236 +239 @@
       <SubType>Code</SubType>
     </Compile>
+    <Compile Include="Interfaces\Classification\IClassificationTransformationModel.cs" />
     <Compile Include="Interfaces\Classification\IDiscriminantFunctionThresholdCalculator.cs" />
+    <Compile Include="Interfaces\Clustering\IClusteringTransformationModel.cs" />
     <Compile Include="Interfaces\IDataAnalysisSolutionExporter.cs" />
     <Compile Include="Interfaces\IDataAnalysisTransformation.cs" />
+    <Compile Include="Interfaces\IDataAnalysisTransformationModel.cs" />
     <Compile Include="Interfaces\IDataset.cs" />
     <Compile Include="Interfaces\IDependencyCalculator.cs" />
     <Compile Include="Interfaces\ITransformation.cs" />
-    <Compile Include="Interfaces\Regression\ITransformedRegressionModel.cs" />
-    <Compile Include="Interfaces\Regression\ITransformedRegressionSolution.cs" />
+    <Compile Include="Interfaces\Regression\IRegressionTransformationModel.cs" />
     <Compile Include="Interfaces\Regression\IConfidenceRegressionModel.cs" />
     <Compile Include="Interfaces\Regression\IConfidenceRegressionSolution.cs" />
@@ -256 +261 @@
     <Compile Include="Interfaces\TimeSeriesPrognosis\ITimeSeriesPrognosisProblemData.cs" />
     <Compile Include="Interfaces\TimeSeriesPrognosis\ITimeSeriesPrognosisSolution.cs" />
+    <Compile Include="Interfaces\TimeSeriesPrognosis\ITimeSeriesPrognosisTransformationModel.cs" />
     <Compile Include="ModifiableDataset.cs" />
     <Compile Include="OnlineCalculators\AutoCorrelationCalculator.cs" />

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblemData.cs

@@ -243 +243 @@
 
     public virtual IDataAnalysisProblemData Transform() {
-      var newDataset = Transform(Dataset, Transformations);
+      var newDataset = DataAnalysisTransformationModel.Transform(Dataset, Transformations);
 
       var extendedInputs = ExtendInputVariables(AllowedInputVariables, Transformations);
@@ -253 +253 @@
       cloner.RegisterClonedObject(Dataset, newDataset);
       cloner.RegisterClonedObject(InputVariables, checkedInputs.AsReadOnly());
-      //cloner.RegisterClonedObject(TransformationsParameter.Value, new ItemList<IDataAnalysisTransformation>(transformations).AsReadOnly());
+      // TODO: valid values for target are not extended
 
       return cloner.Clone(this);
@@ -268 +268 @@
       cloner.RegisterClonedObject(Dataset, newDataset);
       cloner.RegisterClonedObject(InputVariables, checkedInputs.AsReadOnly());
-      //cloner.RegisterClonedObject(TransformationsParameter.Value, new ItemList<IDataAnalysisTransformation>().AsReadOnly());
+      // TODO: check valid target values 
 
       return cloner.Clone(this);
-    }
-
-    public static IDataset Transform(IDataset dataset, IEnumerable<IDataAnalysisTransformation> transformations) {
-      var modifiableDataset = ((Dataset)dataset).ToModifiable();
-
-      foreach (var transformation in transformations) {
-        var trans = (ITransformation<double>)transformation.Transformation;
-
-        var originalData = modifiableDataset.GetDoubleValues(transformation.OriginalVariable);
-        if (!trans.Check(originalData, out string errorMessage))
-          throw new InvalidOperationException($"Cannot estimate Values, Transformation is invalid: {errorMessage}");
-
-        var transformedData = trans.Apply(originalData).ToList();
-        if (modifiableDataset.VariableNames.Contains(transformation.TransformedVariable))
-          modifiableDataset.ReplaceVariable(transformation.TransformedVariable, transformedData);
-        else
-          modifiableDataset.AddVariable(transformation.TransformedVariable, transformedData);
-      }
-
-      return modifiableDataset;
     }
 

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisTransformation.cs

@@ -20 +20 @@
 #endregion
 
-using System;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -40 +39 @@
     }
 
-    private IValueParameter<ITransformation> TransformationParameter {
-      get { return (IValueParameter<ITransformation>)Parameters["Transformation"]; }
+    private ValueParameter<ITransformation> TransformationParameter {
+      get { return (ValueParameter<ITransformation>)Parameters["Transformation"]; }
     }
     #endregion
@@ -48 +47 @@
     public string OriginalVariable {
       get { return OriginalVariableParameter.Value.Value; }
-      set { OriginalVariableParameter.Value.Value = value; }
     }
 
     public string TransformedVariable {
       get { return TransformedVariableParameter.Value.Value; }
-      set { TransformedVariableParameter.Value.Value = value; }
     }
 
     public ITransformation Transformation {
       get { return TransformationParameter.Value; }
-      set { TransformationParameter.Value = value; }
     }
     #endregion
 
     #region Constructor, Cloning & Persistence
-    public DataAnalysisTransformation()
-      : this("", "", new IdentityTransformation()) {
-    }
     public DataAnalysisTransformation(string originalVariable, string transformedVariable, ITransformation transformation)
       : base() {
-      Parameters.Add(new FixedValueParameter<StringValue>("Original Variable", new StringValue(originalVariable)));
-      Parameters.Add(new FixedValueParameter<StringValue>("Transformed Variable", new StringValue(transformedVariable)));
-      Parameters.Add(new ValueParameter<ITransformation>("Transformation", transformation));
-
-      RegisterEventHandlers();
+      Parameters.Add(new FixedValueParameter<StringValue>("Original Variable", new StringValue(originalVariable).AsReadOnly()));
+      Parameters.Add(new FixedValueParameter<StringValue>("Transformed Variable", new StringValue(transformedVariable).AsReadOnly()));
+      Parameters.Add(new ValueParameter<ITransformation>("Transformation", transformation)); // TODO: should be readonly/fixed
     }
 
     protected DataAnalysisTransformation(DataAnalysisTransformation original, Cloner cloner)
       : base(original, cloner) {
-      RegisterEventHandlers();
     }
 
@@ -89 +79 @@
 
     [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() {
-      RegisterEventHandlers();
-    }
-    #endregion
-
-    #region Event-Handling
-    private void RegisterEventHandlers() {
-      OriginalVariableParameter.Value.ValueChanged += OriginalVariableParameterValue_ValueChanged;
-      TransformedVariableParameter.Value.ValueChanged += TransformedVariableParameterValue_ValueChanged;
-      TransformationParameter.ValueChanged += TransformationParameter_ValueChanged;
-    }
-
-    private void OriginalVariableParameterValue_ValueChanged(object sender, EventArgs e) {
-      OnToStringChanged();
-    }
-    private void TransformedVariableParameterValue_ValueChanged(object sender, EventArgs e) {
-      OnToStringChanged();
-    }
-    private void TransformationParameter_ValueChanged(object sender, EventArgs e) {
-      OnToStringChanged();
-    }
     #endregion
 

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisTransformationModel.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
@@ -27 +28 @@
 
 namespace HeuristicLab.Problems.DataAnalysis {
-  [Item("Transformed Regression Model", "A model that was transformed back to match the original variables after the training was performed on transformed variables.")]
+  [Item("Data Analysis Transformation Model", "A model that was transformed back to match the original variables after the training was performed on transformed variables.")]
   [StorableClass]
-  public class TransformedRegressionModel : RegressionModel, ITransformedRegressionModel {
+  public abstract class DataAnalysisTransformationModel : DataAnalysisModel, IDataAnalysisTransformationModel {
 
     [Storable]
-    public IRegressionModel OriginalModel { get; private set; }
+    public IDataAnalysisModel OriginalModel { get; protected set; }
 
     [Storable]
-    public ItemList<IDataAnalysisTransformation> Transformations { get; private set; }
+    public ReadOnlyItemList<IDataAnalysisTransformation> InputTransformations { get; protected set; }
+
+    [Storable]
+    public ReadOnlyItemList<IDataAnalysisTransformation> TargetTransformations { get; protected set; }
+
+    // Usually, the TargetVariable is usually only implemented for Regression and Classification.
+    // However, we implement it in the base class for reducing code duplication and to avoid quasi-identical views.
+    [Storable]
+    private string targetVariable;
+    public string TargetVariable {
+      get { return targetVariable; }
+      set {
+        if (string.IsNullOrEmpty(value) || targetVariable == value) return;
+        targetVariable = value;
+        OnTargetVariableChanged(this, EventArgs.Empty);
+      }
+    }
 
     public override IEnumerable<string> VariablesUsedForPrediction {
-      get { return OriginalModel.VariablesUsedForPrediction; }
+      get { return OriginalModel.VariablesUsedForPrediction; /* TODO: reduce extend-inputs */}
     }
 
     #region Constructor, Cloning & Persistence
-    public TransformedRegressionModel(IRegressionModel originalModel, IEnumerable<IDataAnalysisTransformation> transformations)
-      : base(RegressionProblemData.GetOriginalTragetVariable(originalModel.TargetVariable, transformations)) {
-      Name = "Transformed " + originalModel.Name;
+    protected DataAnalysisTransformationModel(IDataAnalysisModel originalModel, IEnumerable<IDataAnalysisTransformation> transformations)
+      : base(originalModel.Name) {
       OriginalModel = originalModel;
-      Transformations = new ItemList<IDataAnalysisTransformation>(transformations);
+      var transitiveInputs = CalculateTransitiveVariables(originalModel.VariablesUsedForPrediction, transformations);
+      InputTransformations = new ItemList<IDataAnalysisTransformation>(transformations.Where(t => transitiveInputs.Contains(t.OriginalVariable))).AsReadOnly();
+      TargetTransformations = new ReadOnlyItemList<IDataAnalysisTransformation>();
     }
 
-    protected TransformedRegressionModel(TransformedRegressionModel original, Cloner cloner)
+    protected DataAnalysisTransformationModel(DataAnalysisTransformationModel original, Cloner cloner)
       : base(original, cloner) {
       OriginalModel = cloner.Clone(original.OriginalModel);
-      Transformations = cloner.Clone(original.Transformations);
-    }
-
-    public override IDeepCloneable Clone(Cloner cloner) {
-      return new TransformedRegressionModel(this, cloner);
+      InputTransformations = cloner.Clone(original.InputTransformations);
+      TargetTransformations = cloner.Clone(original.TargetTransformations);
+      targetVariable = original.targetVariable;
     }
 
     [StorableConstructor]
-    protected TransformedRegressionModel(bool deserializing)
+    protected DataAnalysisTransformationModel(bool deserializing)
       : base(deserializing) { }
     #endregion
 
+    public static ISet<string> CalculateTransitiveVariables(IEnumerable<string> inputVariables, IEnumerable<IDataAnalysisTransformation> transformations) {
+      var transitiveInputs = new HashSet<string>(inputVariables);
+
+      foreach (var transformation in transformations.Reverse()) {
+        if (transitiveInputs.Contains(transformation.TransformedVariable)) {
+          transitiveInputs.Add(transformation.OriginalVariable);
+        }
+      }
+
+      return transitiveInputs;
+    }
+
+    public static IDataset Transform(IDataset dataset, IEnumerable<IDataAnalysisTransformation> transformations) {
+      var modifiableDataset = ((Dataset)dataset).ToModifiable();
+
+      foreach (var transformation in transformations) {
+        var trans = (ITransformation<double>)transformation.Transformation;
+
+        var originalData = modifiableDataset.GetDoubleValues(transformation.OriginalVariable);
+        if (!trans.Check(originalData, out string errorMessage))
+          throw new InvalidOperationException($"Cannot estimate Values, Transformation is invalid: {errorMessage}");
+
+        var transformedData = trans.Apply(originalData).ToList();
+        if (modifiableDataset.VariableNames.Contains(transformation.TransformedVariable))
+          modifiableDataset.ReplaceVariable(transformation.TransformedVariable, transformedData);
+        else
+          modifiableDataset.AddVariable(transformation.TransformedVariable, transformedData);
+      }
+
+      return modifiableDataset;
+    }
+
+
+
+    #region Events
+    public event EventHandler TargetVariableChanged;
+    private void OnTargetVariableChanged(object sender, EventArgs args) {
+      var changed = TargetVariableChanged;
+      if (changed != null)
+        changed(sender, args);
+    }
+    #endregion
+
+
+
+
+
+
+    /*
     // dataset in original data range
     public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      var transformedDataset = TransformInputs(dataset, Transformations);
+      var transformedDataset = Transform(dataset, Transformations);
 
       var estimates = OriginalModel.GetEstimatedValues(transformedDataset, rows);
 
-      return InverseTransformEstimates(estimates, Transformations, OriginalModel.TargetVariable);
+      return InverseTransform(estimates, Transformations, OriginalModel.TargetVariable);
     }
 
@@ -76 +141 @@
     public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
       // TODO: specialized views for the original solution type are lost (RandomForestSolutionView, ...)
-      return new TransformedRegressionSolution(this, new RegressionProblemData(problemData));
-    }
-
-    private static IDataset TransformInputs(IDataset dataset, IEnumerable<IDataAnalysisTransformation> transformations) {
-      return DataAnalysisProblemData.Transform(dataset, transformations);
-    }
-
-    private static IEnumerable<double> InverseTransformEstimates(IEnumerable<double> data, IEnumerable<IDataAnalysisTransformation> transformations, string targetVariable) {
-      var estimates = data.ToList();
-
-      foreach (var transformation in transformations.Reverse()) {
-        if (transformation.TransformedVariable == targetVariable) {
-          var trans = (ITransformation<double>)transformation.Transformation;
-
-          estimates = trans.InverseApply(estimates).ToList();
-
-          // setup next iteration
-          targetVariable = transformation.OriginalVariable;
-        }
-      }
-
-      return estimates;
-    }
+      return new RegressionSolution(this, new RegressionProblemData(problemData));
+    } */
   }
 }

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblemData.cs

@@ -183 +183 @@
     public override IDataAnalysisProblemData Transform() {
       var transformedProblemData = (IRegressionProblemData)base.Transform();
-      var targetVariable = GetTransformedTragetVariable(TargetVariable, Transformations);
+      var targetVariable = RegressionTransformationModel.GetTransformedTragetVariable(TargetVariable, Transformations);
       transformedProblemData.TargetVariable = targetVariable;
       return transformedProblemData;
@@ -190 +190 @@
     public override IDataAnalysisProblemData InverseTransform() {
       var transformedProblemData = (IRegressionProblemData)base.InverseTransform();
-      var targetVariable = GetOriginalTragetVariable(TargetVariable, Transformations);
+      var targetVariable = RegressionTransformationModel.GetOriginalTragetVariable(TargetVariable, Transformations);
       transformedProblemData.TargetVariable = targetVariable;
       return transformedProblemData;
     }
-
-    public static string GetTransformedTragetVariable(string originalTarget, IEnumerable<IDataAnalysisTransformation> transformations) {
-      var transformedTarget = originalTarget;
-      foreach (var transformation in transformations) {
-        if (transformation.OriginalVariable == transformedTarget)
-          transformedTarget = transformation.TransformedVariable;
-      }
-      return transformedTarget;
-    }
-
-    public static string GetOriginalTragetVariable(string transformedTarget, IEnumerable<IDataAnalysisTransformation> transformations) {
-      var originalTarget = transformedTarget;
-      foreach (var transformation in transformations.Reverse()) {
-        if (transformation.TransformedVariable == originalTarget)
-          originalTarget = transformation.OriginalVariable;
-      }
-      return originalTarget;
-    }
   }
 }

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/IDataAnalysisTransformation.cs

@@ -25 +25 @@
   public interface IDataAnalysisTransformation : INamedItem {
 
-    string OriginalVariable { get; set; }
-    string TransformedVariable { get; set; }
+    string OriginalVariable { get; }
+    string TransformedVariable { get; }
 
-    ITransformation Transformation { get; set; }
+    ITransformation Transformation { get; }
   }
 }

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/Regression/IRegressionTransformationModel.cs

@@ -23 +23 @@
 
 namespace HeuristicLab.Problems.DataAnalysis {
-  public interface ITransformedRegressionModel : IRegressionModel {
-
-    IRegressionModel OriginalModel { get; }
-
-    ItemList<IDataAnalysisTransformation> Transformations { get; }
+  public interface IRegressionTransformationModel : IDataAnalysisTransformationModel, IRegressionModel {
+    new IRegressionModel OriginalModel { get; }
   }
 }