Changeset 15831 for branches/2904_CalculateImpacts/3.4/Implementation

Timestamp:

03/08/18 10:44:51 (7 years ago)

Author:

fholzing

Message:

#2904: Refactored RegressionSolutionVariableImpactsCalculator. We don't dependent on the solution anymore. The impact can be calculated for a single variable. The calculator can be chosen.

File:

• branches/2904_CalculateImpacts/3.4/Implementation/Regression/RegressionSolutionVariableImpactsCalculator.cs (modified) (7 diffs)

branches/2904_CalculateImpacts/3.4/Implementation/Regression/RegressionSolutionVariableImpactsCalculator.cs

@@ -23 +23 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
@@ -96 +97 @@
     }
 
-    private static void PrepareData(DataPartitionEnum partition,
+    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
       IRegressionSolution solution,
-      out IEnumerable<int> rows,
+      DataPartitionEnum data = DataPartitionEnum.Training,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
+      Func<double, string, bool> progressCallback = null) {
+      return CalculateImpacts(solution.Model, solution.ProblemData, solution.EstimatedValues, data, replacementMethod, factorReplacementMethod, progressCallback);
+    }
+
+    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
+      IRegressionModel model,
+      IRegressionProblemData problemData,
+      IEnumerable<double> estimatedValues,
+      DataPartitionEnum data = DataPartitionEnum.Training,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
+      Func<double, string, bool> progressCallback = null,
+      IOnlineCalculator calculator = null) {
+      //PearsonsRSquared is the default calculator
+      if (calculator == null) { calculator = new OnlinePearsonsRSquaredCalculator(); }
+      IEnumerable<int> rows;
+
+      switch (data) {
+        case DataPartitionEnum.All:
+          rows = problemData.AllIndices;
+          break;
+        case DataPartitionEnum.Test:
+          rows = problemData.TestIndices;
+          break;
+        case DataPartitionEnum.Training:
+          rows = problemData.TrainingIndices;
+          break;
+        default:
+          throw new NotSupportedException("DataPartition not supported");
+      }
+
+      return CalculateImpacts(model, problemData, estimatedValues, rows, calculator, replacementMethod, factorReplacementMethod, progressCallback);
+    }
+
+    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
+     IRegressionModel model,
+     IRegressionProblemData problemData,
+     IEnumerable<double> estimatedValues,
+     IEnumerable<int> rows,
+     IOnlineCalculator calculator,
+     ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+     FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
+     Func<double, string, bool> progressCallback = null) {
+
+      IEnumerable<double> targetValues;
+      double originalValue = -1;
+
+      PrepareData(rows, problemData, estimatedValues, out targetValues, out originalValue, calculator);
+
+      var impacts = new Dictionary<string, double>();
+      var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(model.VariablesUsedForPrediction));
+      var allowedInputVariables = problemData.Dataset.VariableNames.Where(v => inputvariables.Contains(v)).ToList();
+
+      int curIdx = 0;
+      int count = allowedInputVariables
+        .Where(v => problemData.Dataset.VariableHasType<double>(v) || problemData.Dataset.VariableHasType<string>(v))
+        .Count();
+
+      foreach (var inputVariable in allowedInputVariables) {
+        //Report the current progress in percent. If the callback returns true, it means the execution shall be stopped
+        if (progressCallback != null) {
+          curIdx++;
+          if (progressCallback((double)curIdx / count, string.Format("Calculating impact for variable {0} ({1} of {2})", inputVariable, curIdx, count))) { return null; }
+        }
+        impacts[inputVariable] = CalculateImpact(inputVariable, model, problemData.Dataset, rows, targetValues, originalValue, calculator, replacementMethod, factorReplacementMethod);
+      }
+
+      return impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value));
+    }
+
+    public static double CalculateImpact(string variableName,
+      IRegressionSolution solution,
+      IEnumerable<int> rows,
+      IEnumerable<double> targetValues,
+      double originalValue,
+      IOnlineCalculator calculator,
+      DataPartitionEnum data = DataPartitionEnum.Training,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
+      return CalculateImpact(variableName, solution.Model, solution.ProblemData.Dataset, rows, targetValues, originalValue, calculator, replacementMethod, factorReplacementMethod);
+    }
+
+    public static double CalculateImpact(string variableName,
+      IRegressionModel model,
+      IDataset dataset,
+      IEnumerable<int> rows,
+      IEnumerable<double> targetValues,
+      double originalValue,
+      IOnlineCalculator calculator,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
+
+      double impact = 0;
+      var modifiableDataset = ((Dataset)dataset).ToModifiable();
+
+      // calculate impacts for double variables
+      if (dataset.VariableHasType<double>(variableName)) {
+        impact = CalculateImpactForDouble(variableName, model, modifiableDataset, rows, targetValues, originalValue, replacementMethod, calculator);
+      } else if (dataset.VariableHasType<string>(variableName)) {
+        impact = CalculateImpactForString(variableName, model, dataset, modifiableDataset, rows, targetValues, originalValue, factorReplacementMethod, calculator);
+      } else {
+        throw new NotSupportedException("Variable not supported");
+      }
+      return impact;
+    }
+
+    private static void PrepareData(IEnumerable<int> rows,
+      IRegressionProblemData problemData,
+      IEnumerable<double> estimatedValues,
       out IEnumerable<double> targetValues,
-      out double originalR2) {
+      out double originalValue,
+      IOnlineCalculator calculator) {
       OnlineCalculatorError error;
 
-      switch (partition) {
-        case DataPartitionEnum.All:
-          rows = solution.ProblemData.AllIndices;
-          targetValues = solution.ProblemData.TargetVariableValues.ToList();
-          originalR2 = OnlinePearsonsRCalculator.Calculate(solution.ProblemData.TargetVariableValues, solution.EstimatedValues, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation.");
-          originalR2 = originalR2 * originalR2;
-          break;
-        case DataPartitionEnum.Training:
-          rows = solution.ProblemData.TrainingIndices;
-          targetValues = solution.ProblemData.TargetVariableTrainingValues.ToList();
-          originalR2 = solution.TrainingRSquared;
-          break;
-        case DataPartitionEnum.Test:
-          rows = solution.ProblemData.TestIndices;
-          targetValues = solution.ProblemData.TargetVariableTestValues.ToList();
-          originalR2 = solution.TestRSquared;
-          break;
-        default: throw new ArgumentException(string.Format("DataPartition {0} cannot be handled.", partition));
-      }
+      var targetVariableValueList = problemData.TargetVariableValues.ToList();
+      targetValues = rows.Select(v => targetVariableValueList.ElementAt(v));
+      var estimatedValuesPartition = rows.Select(v => estimatedValues.ElementAt(v));
+      originalValue = calculator.CalculateValue(targetValues, estimatedValuesPartition, out error);
+
+      if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation.");
     }
 
     private static double CalculateImpactForDouble(string variableName,
-      IRegressionSolution solution,
+      IRegressionModel model,
       ModifiableDataset modifiableDataset,
       IEnumerable<int> rows,
       IEnumerable<double> targetValues,
-      double originalR2,
-      ReplacementMethodEnum replacementMethod) {
+      double originalValue,
+      ReplacementMethodEnum replacementMethod,
+      IOnlineCalculator calculator) {
       OnlineCalculatorError error;
-      var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, variableName, modifiableDataset, rows, replacementMethod);
-      var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-      if (error != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during R² calculation with replaced inputs."); }
-      return originalR2 - (newR2 * newR2);
+      var newEstimates = EvaluateModelWithReplacedVariable(model, variableName, modifiableDataset, rows, replacementMethod);
+      var newValue = calculator.CalculateValue(targetValues, newEstimates, out error);
+      if (error != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during calculation with replaced inputs."); }
+      return originalValue - newValue;
     }
 
     private static double CalculateImpactForString(string variableName,
-      IRegressionSolution solution,
+      IRegressionModel model,
+      IDataset problemData,
       ModifiableDataset modifiableDataset,
       IEnumerable<int> rows,
       IEnumerable<double> targetValues,
-      double originalR2,
-      FactorReplacementMethodEnum factorReplacementMethod) {
+      double originalValue,
+      FactorReplacementMethodEnum factorReplacementMethod,
+      IOnlineCalculator calculator) {
 
       OnlineCalculatorError error;
@@ -151 +253 @@
         // try replacing with all possible values and find the best replacement value
         var smallestImpact = double.PositiveInfinity;
-        foreach (var repl in solution.ProblemData.Dataset.GetStringValues(variableName, rows).Distinct()) {
-          var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, variableName, modifiableDataset, rows, Enumerable.Repeat(repl, solution.ProblemData.Dataset.Rows));
-          var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
-
-          var curImpact = originalR2 - (newR2 * newR2);
+        foreach (var repl in problemData.GetStringValues(variableName, rows).Distinct()) {
+          var originalValues = modifiableDataset.GetReadOnlyStringValues(variableName).ToList();
+          var newEstimates = EvaluateModelWithReplacedVariable(originalValues, model, variableName, modifiableDataset, rows, Enumerable.Repeat(repl, problemData.Rows).ToList());
+          var newValue = calculator.CalculateValue(targetValues, newEstimates, out error);
+          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs.");
+
+          var curImpact = originalValue - newValue;
           if (curImpact < smallestImpact) smallestImpact = curImpact;
         }
@@ -163 +266 @@
         // for replacement methods shuffle and mode
         // calculate impacts for factor variables
-        var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, variableName, modifiableDataset, rows, factorReplacementMethod);
-        var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-        if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
-
-        return originalR2 - (newR2 * newR2);
-      }
-    }
-    public static double CalculateImpact(string variableName,
-      IRegressionSolution solution,
-      IEnumerable<int> rows,
-      IEnumerable<double> targetValues,
-      double originalR2,
-      DataPartitionEnum data = DataPartitionEnum.Training,
-      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
-      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
-
-      double impact = 0;
-      var modifiableDataset = ((Dataset)solution.ProblemData.Dataset).ToModifiable();
-
-      // calculate impacts for double variables
-      if (solution.ProblemData.Dataset.VariableHasType<double>(variableName)) {
-        impact = CalculateImpactForDouble(variableName, solution, modifiableDataset, rows, targetValues, originalR2, replacementMethod);
-      } else if (solution.ProblemData.Dataset.VariableHasType<string>(variableName)) {
-        impact = CalculateImpactForString(variableName, solution, modifiableDataset, rows, targetValues, originalR2, factorReplacementMethod);
-      } else {
-        throw new NotSupportedException("Variable not supported");
-      }
-      return impact;
-    }
-
-    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
-      IRegressionSolution solution,
-      DataPartitionEnum data = DataPartitionEnum.Training,
-      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
-      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
-      Func<double, string, bool> progressCallback = null) {
-
-      IEnumerable<int> rows;
-      IEnumerable<double> targetValues;
-      double originalR2 = -1;
-
-      PrepareData(data, solution, out rows, out targetValues, out originalR2);
-
-      var impacts = new Dictionary<string, double>();
-      var inputvariables = new HashSet<string>(solution.ProblemData.AllowedInputVariables.Union(solution.Model.VariablesUsedForPrediction));
-      var allowedInputVariables = solution.ProblemData.Dataset.VariableNames.Where(v => inputvariables.Contains(v)).ToList();
-
-      int curIdx = 0;
-      int count = allowedInputVariables.Where(solution.ProblemData.Dataset.VariableHasType<double>).Count();
-      // calculate impacts for double variables
-      foreach (var inputVariable in allowedInputVariables) {
-        //Report the current progress in percent. If the callback returns true, it means the execution shall be stopped
-        if (progressCallback != null) {
-          curIdx++;
-          if (progressCallback((double)curIdx / count, string.Format("Calculating impact for variable {0} ({1} of {2})", inputVariable, curIdx, count))) { return null; }
-        }
-        impacts[inputVariable] = CalculateImpact(inputVariable, solution, rows, targetValues, originalR2, data, replacementMethod, factorReplacementMethod);
-      }
-
-      return impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value));
+        var newEstimates = EvaluateModelWithReplacedVariable(model, variableName, modifiableDataset, rows, factorReplacementMethod);
+        var newValue = calculator.CalculateValue(targetValues, newEstimates, out error);
+        if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs.");
+
+        return originalValue - newValue;
+      }
     }
 
@@ -269 +318 @@
       }
 
-      return EvaluateModelWithReplacedVariable(model, variable, dataset, rows, replacementValues);
+      return EvaluateModelWithReplacedVariable(originalValues, model, variable, dataset, rows, replacementValues);
     }
 
@@ -305 +354 @@
       }
 
-      return EvaluateModelWithReplacedVariable(model, variable, dataset, rows, replacementValues);
-    }
-
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable,
-      ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<double> replacementValues) {
-      var originalValues = dataset.GetReadOnlyDoubleValues(variable).ToList();
-      dataset.ReplaceVariable(variable, replacementValues.ToList());
+      return EvaluateModelWithReplacedVariable(originalValues, model, variable, dataset, rows, replacementValues);
+    }
+
+    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IList originalValues, IRegressionModel model, string variable,
+      ModifiableDataset dataset, IEnumerable<int> rows, IList replacementValues) {
+      dataset.ReplaceVariable(variable, replacementValues);
       //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements
       var estimates = model.GetEstimatedValues(dataset, rows).ToList();
@@ -318 +366 @@
       return estimates;
     }
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable,
-      ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<string> replacementValues) {
-      var originalValues = dataset.GetReadOnlyStringValues(variable).ToList();
-      dataset.ReplaceVariable(variable, replacementValues.ToList());
-      //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements
-      var estimates = model.GetEstimatedValues(dataset, rows).ToList();
-      dataset.ReplaceVariable(variable, originalValues);
-
-      return estimates;
-    }
   }
 }