Changeset 15884 for branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisTransformation.cs

Timestamp:

04/04/18 17:18:02 (6 years ago)

Author:

pfleck

Message:

#2906 Refactoring

• Moved transformation-specific parts out of existing interfaces.
• Moved all Transformation logic to DataAnalysisTransformation.
• Simplified (Inverse)Transformation of Dataset/ProblemData/Model/Solution.

File:

• branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisTransformation.cs (modified) (3 diffs)

branches/2906_Transformations/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisTransformation.cs

@@ -20 +20 @@
 #endregion
 
+using System;
+using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -67 +70 @@
 
     private DataAnalysisTransformation(DataAnalysisTransformation original, Cloner cloner)
-      : base(original, cloner) {
-    }
+      : base(original, cloner) { }
 
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -84 +86 @@
       return $"{Transformation} ({OriginalVariable} -> {TransformedVariable})";
     }
+
+    #region Transformation
+
+    #region Variable Extension & Reduction
+    // originals => include extended
+    public static IEnumerable<string> ExtendVariables(IEnumerable<string> variables, IEnumerable<IDataAnalysisTransformation> transformations) {
+      return GetTransitiveVariables(variables, transformations);
+    }
+
+    // extended => originals
+    public static IEnumerable<string> ReduceVariables(IEnumerable<string> variables, IEnumerable<IDataAnalysisTransformation> transformations) {
+      var originalVariables = new HashSet<string>();
+      foreach (var variable in variables)
+        originalVariables.Add(GetLastTransitiveVariable(variable, transformations, inverse: true));
+      return originalVariables;
+    }
+
+    public static IEnumerable<string> GetTransitiveVariables(IEnumerable<string> variables, IEnumerable<IDataAnalysisTransformation> transformations, bool inverse = false) {
+      var reachableVariables = new HashSet<string>(variables);
+      if (inverse) transformations = transformations.Reverse();
+      foreach (var transformation in transformations) {
+        var source = inverse ? transformation.TransformedVariable : transformation.OriginalVariable;
+        var target = inverse ? transformation.OriginalVariable : transformation.TransformedVariable;
+        if (reachableVariables.Contains(source))
+          reachableVariables.Add(target);
+      }
+
+      return reachableVariables;
+    }
+
+    public static string GetLastTransitiveVariable(string variable, IEnumerable<IDataAnalysisTransformation> transformations, bool inverse = false) {
+      if (inverse) transformations = transformations.Reverse();
+      foreach (var transformation in transformations) {
+        var source = inverse ? transformation.TransformedVariable : transformation.OriginalVariable;
+        var target = inverse ? transformation.OriginalVariable : transformation.TransformedVariable;
+        if (variable == source)
+          variable = target;
+      }
+
+      return variable;
+    }
+    #endregion
+
+    #region Transform Dataset
+    public static IDataset Transform(IDataset dataset, IEnumerable<IDataAnalysisTransformation> transformations) {
+      var modifiableDataset = ((Dataset)dataset).ToModifiable();
+
+      foreach (var transformation in transformations) {
+        var trans = (ITransformation<double>)transformation.Transformation;
+
+        var originalData = modifiableDataset.GetDoubleValues(transformation.OriginalVariable);
+        //if (!trans.Check(originalData, out string errorMessage))
+        //  throw new InvalidOperationException($"Cannot estimate Values, Transformation is invalid: {errorMessage}");
+        // TODO: check was already called before configure (in preprocessing)
+        // TODO: newly specified data might not pass the check but it does not matter because the data is not configured with
+        // e.g. impact calculation -> replacement=most common -> originalMean is zero
+
+        var transformedData = trans.Apply(originalData).ToList();
+        if (modifiableDataset.VariableNames.Contains(transformation.TransformedVariable))
+          modifiableDataset.ReplaceVariable(transformation.TransformedVariable, transformedData);
+        else
+          modifiableDataset.AddVariable(transformation.TransformedVariable, transformedData);
+      }
+
+      return modifiableDataset; // TODO: to regular dataset?
+    }
+
+    public static IDataset InverseTransform(IDataset dataset, IEnumerable<IDataAnalysisTransformation> transformations, bool removeVirtualVariables = true) {
+      var modifiableDataset = ((Dataset)dataset).ToModifiable();
+
+      var transformationsStack = new Stack<IDataAnalysisTransformation>(transformations);
+      while (transformationsStack.Any()) {
+        var transformation = transformationsStack.Pop();
+        var trans = (ITransformation<double>)transformation.Transformation;
+
+        var prevTransformations = transformations.Except(transformationsStack);
+        bool originalWasChanged = prevTransformations.Any(x => x.TransformedVariable == transformation.OriginalVariable);
+        if (originalWasChanged) {
+          var transformedData = modifiableDataset.GetDoubleValues(transformation.TransformedVariable);
+
+          var originalData = trans.InverseApply(transformedData).ToList();
+          modifiableDataset.ReplaceVariable(transformation.OriginalVariable, originalData);
+        }
+      }
+
+      if (removeVirtualVariables) {
+        var originalVariables = ReduceVariables(dataset.VariableNames, transformations);
+        var virtualVariables = dataset.VariableNames.Except(originalVariables);
+        foreach (var virtualVariable in virtualVariables)
+          modifiableDataset.RemoveVariable(virtualVariable);
+      }
+
+      return modifiableDataset; // TODO: to regular dataset?
+    }
+    #endregion
+
+    #region Transform ProblemData
+    public static IDataAnalysisProblemData ApplyTransformations(IDataAnalysisProblemData problemData) {
+      var newDataset = Transform(problemData.Dataset, problemData.Transformations);
+      var extendedInputs = ExtendVariables(problemData.AllowedInputVariables, problemData.Transformations);
+
+      return CreateNewProblemData(problemData, newDataset, extendedInputs, inverse: false);
+    }
+
+    public static IDataAnalysisProblemData InverseApplyTransformations(IDataAnalysisProblemData problemData) {
+      var newDataset = InverseTransform(problemData.Dataset, problemData.Transformations);
+      var reducedInputs = ReduceVariables(problemData.AllowedInputVariables, problemData.Transformations);
+
+      return CreateNewProblemData(problemData, newDataset, reducedInputs, inverse: true);
+    }
+
+    private static IDataAnalysisProblemData CreateNewProblemData(IDataAnalysisProblemData problemData, IDataset dataset, IEnumerable<string> inputs, bool inverse = false) {
+      IDataAnalysisProblemData newProblemData;
+      if (problemData is IRegressionProblemData regressionProblemData) {
+        var newTargetVariable = GetLastTransitiveVariable(regressionProblemData.TargetVariable, problemData.Transformations, inverse);
+        newProblemData = new RegressionProblemData(dataset, inputs, newTargetVariable, problemData.Transformations);
+      } else if (problemData is IClassificationProblemData classificationProblemData) {
+        newProblemData = new ClassificationProblemData(dataset, inputs, classificationProblemData.TargetVariable, problemData.Transformations);
+      } else throw new NotSupportedException("Type of ProblemData not supported");
+
+      newProblemData.TrainingPartition.Start = problemData.TrainingPartition.Start;
+      newProblemData.TrainingPartition.End = problemData.TrainingPartition.End;
+      newProblemData.TestPartition.Start = problemData.TestPartition.Start;
+      newProblemData.TestPartition.End = problemData.TestPartition.End;
+
+      return newProblemData;
+    }
+    #endregion
+
+    #region Transform Model
+    public static IDataAnalysisTransformationModel CreateTransformationIntegratedModel(IDataAnalysisModel model, IEnumerable<IDataAnalysisTransformation> transformations) {
+      if (model is IDataAnalysisTransformationModel)
+        throw new InvalidOperationException("Model already is a transformation model.");
+
+      switch (model) {
+        case ITimeSeriesPrognosisModel timeSeriesPrognosisModel:
+          return new TimeSeriesPrognosisTransformationModel(timeSeriesPrognosisModel, transformations);
+        case IRegressionModel regressionModel:
+          return new RegressionTransformationModel(regressionModel, transformations);
+        case IClassificationModel classificationModel:
+          return new ClassificationTransformationModel(classificationModel, transformations);
+        case IClusteringModel clusteringModel:
+          return new ClusteringTransformationModel(clusteringModel, transformations);
+        default:
+          throw new NotSupportedException("Type of the model is not supported;");
+      }
+    }
+
+    public static IDataAnalysisModel RestoreTrainedModel(IDataAnalysisModel transformationModel, IEnumerable<IDataAnalysisTransformation> transformations) {
+      if (!(transformationModel is IDataAnalysisTransformationModel model))
+        throw new InvalidOperationException("Cannot restore because model is not a TransformationModel");
+      return model.OriginalModel;
+    }
+    #endregion
+
+    #region Transform Solution
+    public static IDataAnalysisSolution TransformSolution(IDataAnalysisSolution solution) {
+      var transformations = solution.ProblemData.Transformations;
+
+      var model = solution.Model is IDataAnalysisTransformationModel // TODO: what if model is a integrated sym-reg model?
+        ? RestoreTrainedModel(solution.Model, transformations)
+        : CreateTransformationIntegratedModel(solution.Model, transformations);
+
+      var data = solution.Model is IDataAnalysisTransformationModel
+        ? ApplyTransformations(solution.ProblemData) // original -> transformed
+        : InverseApplyTransformations(solution.ProblemData); // transformed -> original
+
+      return CreateSolution(model, data);
+    }
+
+    private static IDataAnalysisSolution CreateSolution(IDataAnalysisModel model, IDataAnalysisProblemData problemData) {
+      switch (model) {
+        case ITimeSeriesPrognosisModel timeSeriesPrognosisModel:
+          return timeSeriesPrognosisModel.CreateTimeSeriesPrognosisSolution((ITimeSeriesPrognosisProblemData)problemData);
+        case IRegressionModel regressionModel:
+          return regressionModel.CreateRegressionSolution((IRegressionProblemData)problemData);
+        case IClassificationModel classificationModel:
+          return classificationModel.CreateClassificationSolution((IClassificationProblemData)problemData);
+        default:
+          throw new NotSupportedException("Cannot create Solution of the model type.");
+      }
+    }
+    #endregion
+
+    #endregion
   }
 }