Changeset 16055

Timestamp:

08/06/18 17:35:11 (6 years ago)

Author:

fholzing

Message:

#2904: Also applied changes from regression to classification

Location:

branches/2904_CalculateImpacts

Files:

• 3.4/Implementation/Classification/ClassificationSolutionVariableImpactsCalculator.cs (modified) (4 diffs)
• HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionVariableImpactsView.cs (modified) (6 diffs)

branches/2904_CalculateImpacts/3.4/Implementation/Classification/ClassificationSolutionVariableImpactsCalculator.cs

@@ -90 +90 @@
     public ClassificationSolutionVariableImpactsCalculator()
       : base() {
-      Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum.Median)));
+      Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum.Shuffle)));
       Parameters.Add(new FixedValueParameter<EnumValue<FactorReplacementMethodEnum>>(FactorReplacementParameterName, "The replacement method for factor variables during impact calculation.", new EnumValue<FactorReplacementMethodEnum>(FactorReplacementMethodEnum.Best)));
       Parameters.Add(new FixedValueParameter<EnumValue<DataPartitionEnum>>(DataPartitionParameterName, "The data partition on which the impacts are calculated.", new EnumValue<DataPartitionEnum>(DataPartitionEnum.Training)));
@@ -110 +110 @@
       FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
       DataPartitionEnum dataPartition = DataPartitionEnum.Training) {
-      return CalculateImpacts(solution.Model, solution.ProblemData, solution.EstimatedClassValues, replacementMethod, factorReplacementMethod, dataPartition);
-    }
-
-    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
-      IClassificationModel model,
-      IClassificationProblemData problemData,
-      IEnumerable<double> estimatedValues,
-      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
-      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
-      DataPartitionEnum dataPartition = DataPartitionEnum.Training) {
-      IEnumerable<int> rows = GetPartitionRows(dataPartition, problemData);
-      return CalculateImpacts(model, problemData, estimatedValues, rows, replacementMethod, factorReplacementMethod);
+
+      IEnumerable<int> rows = GetPartitionRows(dataPartition, solution.ProblemData);
+      IEnumerable<double> estimatedClassValues = solution.GetEstimatedClassValues(rows);
+      return CalculateImpacts(solution.Model, solution.ProblemData, estimatedClassValues, rows, replacementMethod, factorReplacementMethod);
     }
 
@@ -131 +123 @@
      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
-      //Calculate original quality-values (via calculator, default is Accuracy)    
-      OnlineCalculatorError error;
-      IEnumerable<double> targetValuesPartition = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
-      IEnumerable<double> estimatedValuesPartition = rows.Select(v => estimatedClassValues.ElementAt(v));
-      var originalCalculatorValue = CalculateVariableImpact(targetValuesPartition, estimatedValuesPartition, out error);
-      if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation.");
+
+      //fholzing: try and catch in case a different dataset is loaded, otherwise statement is neglectable
+      var missingVariables = model.VariablesUsedForPrediction.Except(problemData.Dataset.VariableNames);
+      if (missingVariables.Any()) {
+        throw new InvalidOperationException(string.Format("Can not calculate variable impacts, because the model uses inputs missing in the dataset ({0})", string.Join(", ", missingVariables)));
+      }
+      IEnumerable<double> targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      var originalQuality = CalculateQuality(targetValues, estimatedClassValues);
 
       var impacts = new Dictionary<string, double>();
       var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(model.VariablesUsedForPrediction));
-      var allowedInputVariables = problemData.Dataset.VariableNames.Where(v => inputvariables.Contains(v)).ToList();
       var modifiableDataset = ((Dataset)(problemData.Dataset).Clone()).ToModifiable();
 
-      foreach (var inputVariable in allowedInputVariables) {
-        if (model.VariablesUsedForPrediction.Contains(inputVariable)) {
-          impacts[inputVariable] = CalculateImpact(inputVariable, model, modifiableDataset, rows, targetValuesPartition, originalCalculatorValue, replacementMethod, factorReplacementMethod);
-        } else {
-          impacts[inputVariable] = 0;
-        }
-      }
-
-      return impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value));
+      foreach (var inputVariable in inputvariables) {
+        impacts[inputVariable] = CalculateImpact(inputVariable, model, problemData, modifiableDataset, rows, replacementMethod, factorReplacementMethod, targetValues, originalQuality);
+      }
+
+      return impacts.Select(i => Tuple.Create(i.Key, i.Value));
     }
 
     public static double CalculateImpact(string variableName,
       IClassificationModel model,
+      IClassificationProblemData problemData,
       ModifiableDataset modifiableDataset,
       IEnumerable<int> rows,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
+      IEnumerable<double> targetValues = null,
+      double quality = double.NaN) {
+
+      if (!model.VariablesUsedForPrediction.Contains(variableName)) { return 0.0; }
+      if (!problemData.Dataset.VariableNames.Contains(variableName)) {
+        throw new InvalidOperationException(string.Format("Can not calculate variable impact, because the model uses inputs missing in the dataset ({0})", variableName));
+      }
+
+      if (targetValues == null) {
+        targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      }
+      if (quality == double.NaN) {
+        quality = CalculateQuality(model.GetEstimatedClassValues(modifiableDataset, rows), targetValues);
+      }
+
+      IList originalValues = null;
+      IList replacementValues = GetReplacementValues(modifiableDataset, variableName, model, rows, targetValues, out originalValues, replacementMethod, factorReplacementMethod);
+
+      double newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, replacementValues, targetValues);
+      double impact = quality - newValue;
+
+      return impact;
+    }
+
+    private static IList GetReplacementValues(ModifiableDataset modifiableDataset,
+      string variableName,
+      IClassificationModel model,
+      IEnumerable<int> rows,
       IEnumerable<double> targetValues,
-      double originalValue,
+      out IList originalValues,
       ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
       FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
-      double impact = 0;
-      OnlineCalculatorError error;
-      IRandom random;
-      double replacementValue;
-      IEnumerable<double> newEstimates = null;
-      double newValue = 0;
-
+
+      IList replacementValues = null;
       if (modifiableDataset.VariableHasType<double>(variableName)) {
-        #region NumericalVariable
-        var originalValues = modifiableDataset.GetReadOnlyDoubleValues(variableName).ToList();
-        List<double> replacementValues;
-
-        switch (replacementMethod) {
-          case ReplacementMethodEnum.Median:
-            replacementValue = rows.Select(r => originalValues[r]).Median();
-            replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
-            break;
-          case ReplacementMethodEnum.Average:
-            replacementValue = rows.Select(r => originalValues[r]).Average();
-            replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
-            break;
-          case ReplacementMethodEnum.Shuffle:
-            // new var has same empirical distribution but the relation to y is broken
-            random = new FastRandom(31415);
-            // prepare a complete column for the dataset
-            replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
-            // shuffle only the selected rows
-            var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
-            int i = 0;
-            // update column values 
-            foreach (var r in rows) {
-              replacementValues[r] = shuffledValues[i++];
-            }
-            break;
-          case ReplacementMethodEnum.Noise:
-            var avg = rows.Select(r => originalValues[r]).Average();
-            var stdDev = rows.Select(r => originalValues[r]).StandardDeviation();
-            random = new FastRandom(31415);
-            // prepare a complete column for the dataset
-            replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
-            // update column values 
-            foreach (var r in rows) {
-              replacementValues[r] = NormalDistributedRandom.NextDouble(random, avg, stdDev);
-            }
-            break;
-
-          default:
-            throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacementMethod));
-        }
-
-        newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, replacementValues);
-        newValue = CalculateVariableImpact(targetValues, newEstimates, out error);
-        if (error != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during calculation with replaced inputs."); }
-
-        impact = originalValue - newValue;
-        #endregion
+        originalValues = modifiableDataset.GetReadOnlyDoubleValues(variableName).ToList();
+        replacementValues = GetReplacementValuesForDouble(modifiableDataset, rows, (List<double>)originalValues, replacementMethod);
       } else if (modifiableDataset.VariableHasType<string>(variableName)) {
-        #region FactorVariable
-        var originalValues = modifiableDataset.GetReadOnlyStringValues(variableName).ToList();
-        List<string> replacementValues;
-
-        switch (factorReplacementMethod) {
-          case FactorReplacementMethodEnum.Best:
-            // try replacing with all possible values and find the best replacement value
-            var smallestImpact = double.PositiveInfinity;
-            foreach (var repl in modifiableDataset.GetStringValues(variableName, rows).Distinct()) {
-              newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, Enumerable.Repeat(repl, modifiableDataset.Rows).ToList());
-              newValue = CalculateVariableImpact(targetValues, newEstimates, out error);
-              if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs.");
-
-              var curImpact = originalValue - newValue;
-              if (curImpact < smallestImpact) smallestImpact = curImpact;
-            }
-            impact = smallestImpact;
-            break;
-          case FactorReplacementMethodEnum.Mode:
-            var mostCommonValue = rows.Select(r => originalValues[r])
-              .GroupBy(v => v)
-              .OrderByDescending(g => g.Count())
-              .First().Key;
-            replacementValues = Enumerable.Repeat(mostCommonValue, modifiableDataset.Rows).ToList();
-
-            newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, replacementValues);
-            newValue = CalculateVariableImpact(targetValues, newEstimates, out error);
-            if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs.");
-
-            impact = originalValue - newValue;
-            break;
-          case FactorReplacementMethodEnum.Shuffle:
-            // new var has same empirical distribution but the relation to y is broken
-            random = new FastRandom(31415);
-            // prepare a complete column for the dataset
-            replacementValues = Enumerable.Repeat(string.Empty, modifiableDataset.Rows).ToList();
-            // shuffle only the selected rows
-            var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
-            int i = 0;
-            // update column values 
-            foreach (var r in rows) {
-              replacementValues[r] = shuffledValues[i++];
-            }
-
-            newEstimates = GetReplacedEstimates(originalValues, model, variableName, modifiableDataset, rows, replacementValues);
-            newValue = CalculateVariableImpact(targetValues, newEstimates, out error);
-            if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation with replaced inputs.");
-
-            impact = originalValue - newValue;
-            break;
-          default:
-            throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", factorReplacementMethod));
-        }
-        #endregion
+        originalValues = modifiableDataset.GetReadOnlyStringValues(variableName).ToList();
+        replacementValues = GetReplacementValuesForString(model, modifiableDataset, variableName, rows, originalValues, targetValues, factorReplacementMethod);
       } else {
         throw new NotSupportedException("Variable not supported");
       }
 
-      return impact;
-    }
-
-    /// <summary>
-    /// Replaces the values of the original model-variables with the replacement variables, calculates the new estimated values 
-    /// and changes the value of the model-variables back to the original ones.
-    /// </summary>
-    /// <param name="originalValues"></param>
-    /// <param name="model"></param>
-    /// <param name="variableName"></param>
-    /// <param name="modifiableDataset"></param>
-    /// <param name="rows"></param>
-    /// <param name="replacementValues"></param>
-    /// <returns></returns>
-    private static IEnumerable<double> GetReplacedEstimates(
+      return replacementValues;
+    }
+
+    private static IList GetReplacementValuesForDouble(ModifiableDataset modifiableDataset,
+      IEnumerable<int> rows,
+      List<double> originalValues,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle) {
+
+      IRandom random = new FastRandom(31415);
+      List<double> replacementValues;
+      double replacementValue;
+
+      switch (replacementMethod) {
+        case ReplacementMethodEnum.Median:
+          replacementValue = rows.Select(r => originalValues[r]).Median();
+          replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
+          break;
+        case ReplacementMethodEnum.Average:
+          replacementValue = rows.Select(r => originalValues[r]).Average();
+          replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
+          break;
+        case ReplacementMethodEnum.Shuffle:
+          // new var has same empirical distribution but the relation to y is broken
+          // prepare a complete column for the dataset
+          replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
+          // shuffle only the selected rows
+          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
+          int i = 0;
+          // update column values 
+          foreach (var r in rows) {
+            replacementValues[r] = shuffledValues[i++];
+          }
+          break;
+        case ReplacementMethodEnum.Noise:
+          var avg = rows.Select(r => originalValues[r]).Average();
+          var stdDev = rows.Select(r => originalValues[r]).StandardDeviation();
+          // prepare a complete column for the dataset
+          replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
+          // update column values 
+          foreach (var r in rows) {
+            replacementValues[r] = NormalDistributedRandom.NextDouble(random, avg, stdDev);
+          }
+          break;
+
+        default:
+          throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacementMethod));
+      }
+
+      return replacementValues;
+    }
+
+    private static IList GetReplacementValuesForString(IClassificationModel model,
+      ModifiableDataset modifiableDataset,
+      string variableName,
+      IEnumerable<int> rows,
       IList originalValues,
+      IEnumerable<double> targetValues,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Shuffle) {
+
+      IList replacementValues = null;
+      IRandom random = new FastRandom(31415);
+
+      switch (factorReplacementMethod) {
+        case FactorReplacementMethodEnum.Best:
+          // try replacing with all possible values and find the best replacement value
+          var bestQuality = double.NegativeInfinity;
+          foreach (var repl in modifiableDataset.GetStringValues(variableName, rows).Distinct()) {
+            List<string> curReplacementValues = Enumerable.Repeat(repl, modifiableDataset.Rows).ToList();
+            //fholzing: this result could be used later on (theoretically), but is neglected for better readability/method consistency 
+            var newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, curReplacementValues, targetValues);
+            var curQuality = newValue;
+
+            if (curQuality > bestQuality) {
+              bestQuality = curQuality;
+              replacementValues = curReplacementValues;
+            }
+          }
+          break;
+        case FactorReplacementMethodEnum.Mode:
+          var mostCommonValue = rows.Select(r => originalValues[r])
+            .GroupBy(v => v)
+            .OrderByDescending(g => g.Count())
+            .First().Key;
+          replacementValues = Enumerable.Repeat(mostCommonValue, modifiableDataset.Rows).ToList();
+          break;
+        case FactorReplacementMethodEnum.Shuffle:
+          // new var has same empirical distribution but the relation to y is broken
+          // prepare a complete column for the dataset
+          replacementValues = Enumerable.Repeat(string.Empty, modifiableDataset.Rows).ToList();
+          // shuffle only the selected rows
+          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
+          int i = 0;
+          // update column values 
+          foreach (var r in rows) {
+            replacementValues[r] = shuffledValues[i++];
+          }
+          break;
+        default:
+          throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", factorReplacementMethod));
+      }
+
+      return replacementValues;
+    }
+
+    private static double CalculateQualityForReplacement(
       IClassificationModel model,
+      ModifiableDataset modifiableDataset,
       string variableName,
-      ModifiableDataset modifiableDataset,
-      IEnumerable<int> rows,
-      IList replacementValues) {
+      IList originalValues,
+      IEnumerable<int> rows,
+      IList replacementValues,
+      IEnumerable<double> targetValues) {
+
       modifiableDataset.ReplaceVariable(variableName, replacementValues);
-
       var discModel = model as IDiscriminantFunctionClassificationModel;
       if (discModel != null) {
@@ -308 +316 @@
       //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements
       var estimates = model.GetEstimatedClassValues(modifiableDataset, rows).ToList();
+      var ret = CalculateQuality(targetValues, estimates);
       modifiableDataset.ReplaceVariable(variableName, originalValues);
 
-      return estimates;
-    }
-
-    /// <summary>
-    /// Calculates and returns the VariableImpact (calculated via Accuracy).
-    /// </summary>
-    /// <param name="targetValues">The actual values</param>
-    /// <param name="estimatedValues">The calculated/replaced values</param>
-    /// <param name="errorState"></param>
-    /// <returns></returns>
-    public static double CalculateVariableImpact(IEnumerable<double> targetValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
-      //Theoretically, all calculators implement a static Calculate-Method which provides the same functionality
-      //as the code below does. But this way we can easily swap the calculator later on, so the user  
-      //could choose a Calculator during runtime in future versions.
-      IOnlineCalculator calculator = new OnlineAccuracyCalculator();
-      IEnumerator<double> firstEnumerator = targetValues.GetEnumerator();
-      IEnumerator<double> secondEnumerator = estimatedValues.GetEnumerator();
-
-      // always move forward both enumerators (do not use short-circuit evaluation!)
-      while (firstEnumerator.MoveNext() & secondEnumerator.MoveNext()) {
-        double original = firstEnumerator.Current;
-        double estimated = secondEnumerator.Current;
-        calculator.Add(original, estimated);
-        if (calculator.ErrorState != OnlineCalculatorError.None) break;
-      }
-
-      // check if both enumerators are at the end to make sure both enumerations have the same length
-      if (calculator.ErrorState == OnlineCalculatorError.None &&
-           (secondEnumerator.MoveNext() || firstEnumerator.MoveNext())) {
-        throw new ArgumentException("Number of elements in first and second enumeration doesn't match.");
-      } else {
-        errorState = calculator.ErrorState;
-        return calculator.Value;
-      }
-    }
-
-    /// <summary>
-    /// Returns a collection of the row-indices for a given DataPartition (training or test)
-    /// </summary>
-    /// <param name="dataPartition"></param>
-    /// <param name="problemData"></param>
-    /// <returns></returns>
+      return ret;
+    }
+
+    public static double CalculateQuality(IEnumerable<double> targetValues, IEnumerable<double> estimatedClassValues) {
+      OnlineCalculatorError errorState;
+      var ret = OnlineAccuracyCalculator.Calculate(targetValues, estimatedClassValues, out errorState);
+      if (errorState != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during calculation with replaced inputs."); }
+      return ret;
+    }
+
     public static IEnumerable<int> GetPartitionRows(DataPartitionEnum dataPartition, IClassificationProblemData problemData) {
       IEnumerable<int> rows;

branches/2904_CalculateImpacts/HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionVariableImpactsView.cs

@@ -107 +107 @@
 
     private async void UpdateVariableImpact() {
+      IProgress progress;
+
       //Check if the selection is valid
       if (Content == null) { return; }
@@ -112 +114 @@
       if (dataPartitionComboBox.SelectedIndex < 0) { return; }
       if (factorVarReplComboBox.SelectedIndex < 0) { return; }
-
-      IProgress progress;
 
       //Prepare arguments
@@ -137 +137 @@
 
         List<Tuple<string, double>> impacts = null;
-
-        await Task.Run(() => { impacts = CalculateVariableImpacts(originalVariableOrdering, (IClassificationModel)Content.Model.Clone(), problemData, Content.EstimatedClassValues, dataPartition, replMethod, factorReplMethod, cancellationToken.Token, progress); });
+        await Task.Run(() => { impacts = CalculateVariableImpacts(originalVariableOrdering, Content.Model, problemData, Content.EstimatedClassValues, dataPartition, replMethod, factorReplMethod, cancellationToken.Token, progress); });
         if (impacts == null) { return; }
 
         rawVariableImpacts.Clear();
-        originalVariableOrdering.ForEach(v => rawVariableImpacts.Add(new Tuple<string, double>(v, impacts.First(vv => vv.Item1 == v).Item2)));
+        rawVariableImpacts.AddRange(impacts);
         UpdateOrdering();
       }
@@ -149 +148 @@
       }
     }
-
     private List<Tuple<string, double>> CalculateVariableImpacts(List<string> originalVariableOrdering,
       IClassificationModel model,
       IClassificationProblemData problemData,
-      IEnumerable<double> estimatedValues,
+      IEnumerable<double> estimatedClassValues,
       ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum dataPartition,
       ClassificationSolutionVariableImpactsCalculator.ReplacementMethodEnum replMethod,
@@ -166 +164 @@
 
       //Calculate original quality-values (via calculator, default is R²)
-      OnlineCalculatorError error;
       IEnumerable<double> targetValuesPartition = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
-      IEnumerable<double> estimatedValuesPartition = Content.GetEstimatedClassValues(rows);
-      var originalCalculatorValue = ClassificationSolutionVariableImpactsCalculator.CalculateVariableImpact(targetValuesPartition, estimatedValuesPartition, out error);
-      if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during calculation.");
-
+      IEnumerable<double> estimatedClassValuesPartition = Content.GetEstimatedClassValues(rows);
+
+      var originalCalculatorValue = ClassificationSolutionVariableImpactsCalculator.CalculateQuality(targetValuesPartition, estimatedClassValuesPartition);
+      var clonedModel = (IClassificationModel)model.Clone();
       foreach (var variableName in originalVariableOrdering) {
         if (cancellationToken.Token.IsCancellationRequested) { return null; }
@@ -180 +177 @@
         //If the variable isn't used for prediction, it has zero impact.
         if (model.VariablesUsedForPrediction.Contains(variableName)) {
-          impact = ClassificationSolutionVariableImpactsCalculator.CalculateImpact(variableName, model, modifiableDataset, rows, targetValuesPartition, originalCalculatorValue, replMethod, factorReplMethod);
+          impact = ClassificationSolutionVariableImpactsCalculator.CalculateImpact(variableName, clonedModel, problemData, modifiableDataset, rows, replMethod, factorReplMethod, targetValuesPartition, originalCalculatorValue);
         }
         impacts.Add(new Tuple<string, double>(variableName, impact));