Changeset 14762 for branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression

Timestamp:

03/18/17 12:47:30 (8 years ago)

Author:

gkronber

Message:

#2650: added option to specify replacement method for factor variables

File:

• branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolutionVariableImpactsCalculator.cs (modified) (10 diffs)

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolutionVariableImpactsCalculator.cs

@@ -42 +42 @@
       Noise
     }
-
+    public enum FactorReplacementMethodEnum {
+      Best,
+      Mode,
+      Shuffle
+    }
     public enum DataPartitionEnum {
       Training,
@@ -88 +92 @@
     }
 
-    public static IEnumerable<Tuple<string, double>> CalculateImpacts(IRegressionSolution solution,
+    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
+      IRegressionSolution solution,
       DataPartitionEnum data = DataPartitionEnum.Training,
-      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median) {
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
 
       var problemData = solution.ProblemData;
@@ -101 +107 @@
       OnlineCalculatorError error;
 
-      switch (data) {
+      switch(data) {
         case DataPartitionEnum.All:
           rows = solution.ProblemData.AllIndices;
           targetValues = problemData.TargetVariableValues.ToList();
           originalR2 = OnlinePearsonsRCalculator.Calculate(problemData.TargetVariableValues, solution.EstimatedValues, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation.");
+          if(error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation.");
           originalR2 = originalR2 * originalR2;
           break;
@@ -129 +135 @@
 
       // calculate impacts for double variables
-      foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<double>)) {
+      foreach(var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<double>)) {
         var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows, replacementMethod);
         var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-        if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
+        if(error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
 
         newR2 = newR2 * newR2;
@@ -138 +144 @@
         impacts[inputVariable] = impact;
       }
-      // calculate impacts for factor variables
-      foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<string>)) {
-        var smallestImpact = double.PositiveInfinity;
-        foreach (var repl in problemData.Dataset.GetStringValues(inputVariable, rows).Distinct()) {
-          var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows, Enumerable.Repeat(repl, dataset.Rows));
+
+      // calculate impacts for string variables
+      foreach(var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<string>)) {
+        if(factorReplacementMethod == FactorReplacementMethodEnum.Best) {
+          // try replacing with all possible values and find the best replacement value
+          var smallestImpact = double.PositiveInfinity;
+          foreach(var repl in problemData.Dataset.GetStringValues(inputVariable, rows).Distinct()) {
+            var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows,
+              Enumerable.Repeat(repl, dataset.Rows));
+            var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
+            if(error != OnlineCalculatorError.None)
+              throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
+
+            newR2 = newR2 * newR2;
+            var impact = originalR2 - newR2;
+            if(impact < smallestImpact) smallestImpact = impact;
+          }
+          impacts[inputVariable] = smallestImpact;
+        } else {
+          // for replacement methods shuffle and mode
+          // calculate impacts for factor variables
+
+          var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows,
+            factorReplacementMethod);
           var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
+          if(error != OnlineCalculatorError.None)
+            throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
 
           newR2 = newR2 * newR2;
           var impact = originalR2 - newR2;
-          if (impact < smallestImpact) smallestImpact = impact;
+          impacts[inputVariable] = impact;
         }
-        impacts[inputVariable] = smallestImpact;
-      }
+      } // foreach
       return impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value));
     }
@@ -161 +186 @@
       IRandom rand;
 
-      switch (replacement) {
+      switch(replacement) {
         case ReplacementMethodEnum.Median:
           replacementValue = rows.Select(r => originalValues[r]).Median();
@@ -179 +204 @@
           int i = 0;
           // update column values 
-          foreach (var r in rows) {
+          foreach(var r in rows) {
             replacementValues[r] = shuffledValues[i++];
           }
@@ -190 +215 @@
           replacementValues = Enumerable.Repeat(double.NaN, dataset.Rows).ToList();
           // update column values 
-          foreach (var r in rows) {
+          foreach(var r in rows) {
             replacementValues[r] = NormalDistributedRandom.NextDouble(rand, avg, stdDev);
           }
@@ -202 +227 @@
     }
 
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable, 
+    private static IEnumerable<double> EvaluateModelWithReplacedVariable(
+      IRegressionModel model, string variable, ModifiableDataset dataset,
+      IEnumerable<int> rows,
+      FactorReplacementMethodEnum replacement = FactorReplacementMethodEnum.Shuffle) {
+      var originalValues = dataset.GetReadOnlyStringValues(variable).ToList();
+      List<string> replacementValues;
+      IRandom rand;
+
+      switch(replacement) {
+        case FactorReplacementMethodEnum.Mode:
+          var mostCommonValue = rows.Select(r => originalValues[r])
+            .GroupBy(v => v)
+            .OrderByDescending(g => g.Count())
+            .First().Key;
+          replacementValues = Enumerable.Repeat(mostCommonValue, dataset.Rows).ToList();
+          break;
+        case FactorReplacementMethodEnum.Shuffle:
+          // new var has same empirical distribution but the relation to y is broken
+          rand = new FastRandom(31415);
+          // prepare a complete column for the dataset
+          replacementValues = Enumerable.Repeat(string.Empty, dataset.Rows).ToList();
+          // shuffle only the selected rows
+          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(rand).ToList();
+          int i = 0;
+          // update column values 
+          foreach(var r in rows) {
+            replacementValues[r] = shuffledValues[i++];
+          }
+          break;
+        default:
+          throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", replacement));
+      }
+
+      return EvaluateModelWithReplacedVariable(model, variable, dataset, rows, replacementValues);
+    }
+
+    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable,
       ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<double> replacementValues) {
       var originalValues = dataset.GetReadOnlyDoubleValues(variable).ToList();
@@ -212 +273 @@
       return estimates;
     }
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable, 
+    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable,
       ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<string> replacementValues) {
       var originalValues = dataset.GetReadOnlyStringValues(variable).ToList();