Changeset 16422

Timestamp:

12/20/18 13:29:51 (6 years ago)

Author:

fholzing

Message:

#2904: reintegrated branch

Location:

trunk

Files:

• HeuristicLab.Problems.DataAnalysis.Views (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Views/3.4 (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionVariableImpactsView.Designer.cs (modified) (6 diffs)
• HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionVariableImpactsView.cs (modified) (9 diffs)
• HeuristicLab.Problems.DataAnalysis.Views/3.4/Regression/RegressionSolutionVariableImpactsView.Designer.cs (modified) (5 diffs)
• HeuristicLab.Problems.DataAnalysis.Views/3.4/Regression/RegressionSolutionVariableImpactsView.cs (modified) (8 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4 (modified) (1 prop)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionVariableImpactsCalculator.cs (modified) (7 diffs)
• HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolutionVariableImpactsCalculator.cs (modified) (7 diffs)
• HeuristicLab.Tests (modified) (1 prop)
• HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/ClassificationVariableImpactCalculationTest.cs (copied) (copied from branches/2904_CalculateImpacts/HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/ClassificationVariableImpactCalculationTest.cs)
• HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/RegressionVariableImpactCalculationTest.cs (copied) (copied from branches/2904_CalculateImpacts/HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/RegressionVariableImpactCalculationTest.cs)
• HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/VariableImpactCalculationTest.cs (deleted)
• HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4/IntervalInterpreterTest.cs (modified) (1 diff)
• HeuristicLab.Tests/HeuristicLab.Tests.csproj (modified) (1 diff)

trunk/HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionVariableImpactsView.Designer.cs

@@ -19 +19 @@
  */
 #endregion
+
+
 namespace HeuristicLab.Problems.DataAnalysis.Views {
   partial class ClassificationSolutionVariableImpactsView {
@@ -44 +46 @@
     /// </summary>
     private void InitializeComponent() {
-      this.variableImactsArrayView = new HeuristicLab.Data.Views.StringConvertibleArrayView();
-      this.dataPartitionComboBox = new System.Windows.Forms.ComboBox();
-      this.dataPartitionLabel = new System.Windows.Forms.Label();
-      this.numericVarReplacementLabel = new System.Windows.Forms.Label();
-      this.replacementComboBox = new System.Windows.Forms.ComboBox();
-      this.factorVarReplacementLabel = new System.Windows.Forms.Label();
-      this.factorVarReplComboBox = new System.Windows.Forms.ComboBox();
       this.ascendingCheckBox = new System.Windows.Forms.CheckBox();
       this.sortByLabel = new System.Windows.Forms.Label();
       this.sortByComboBox = new System.Windows.Forms.ComboBox();
-      this.backgroundWorker = new System.ComponentModel.BackgroundWorker();
+      this.factorVarReplComboBox = new System.Windows.Forms.ComboBox();
+      this.factorVarReplacementLabel = new System.Windows.Forms.Label();
+      this.replacementComboBox = new System.Windows.Forms.ComboBox();
+      this.numericVarReplacementLabel = new System.Windows.Forms.Label();
+      this.dataPartitionLabel = new System.Windows.Forms.Label();
+      this.dataPartitionComboBox = new System.Windows.Forms.ComboBox();
+      this.variableImpactsArrayView = new HeuristicLab.Data.Views.StringConvertibleArrayView();
       this.SuspendLayout();
       // 
-      // variableImactsArrayView
-      // 
-      this.variableImactsArrayView.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
-            | System.Windows.Forms.AnchorStyles.Left) 
-            | System.Windows.Forms.AnchorStyles.Right)));
-      this.variableImactsArrayView.Caption = "StringConvertibleArray View";
-      this.variableImactsArrayView.Content = null;
-      this.variableImactsArrayView.Location = new System.Drawing.Point(3, 84);
-      this.variableImactsArrayView.Name = "variableImactsArrayView";
-      this.variableImactsArrayView.ReadOnly = true;
-      this.variableImactsArrayView.Size = new System.Drawing.Size(662, 278);
-      this.variableImactsArrayView.TabIndex = 2;
-      // 
-      // dataPartitionComboBox
-      // 
-      this.dataPartitionComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
-      this.dataPartitionComboBox.FormattingEnabled = true;
-      this.dataPartitionComboBox.Items.AddRange(new object[] {
-            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum.Training,
-            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum.Test,
-            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum.All});
-      this.dataPartitionComboBox.Location = new System.Drawing.Point(197, 3);
-      this.dataPartitionComboBox.Name = "dataPartitionComboBox";
-      this.dataPartitionComboBox.Size = new System.Drawing.Size(121, 21);
-      this.dataPartitionComboBox.TabIndex = 1;
-      this.dataPartitionComboBox.SelectedIndexChanged += new System.EventHandler(this.dataPartitionComboBox_SelectedIndexChanged);
-      // 
-      // dataPartitionLabel
-      // 
-      this.dataPartitionLabel.AutoSize = true;
-      this.dataPartitionLabel.Location = new System.Drawing.Point(3, 6);
-      this.dataPartitionLabel.Name = "dataPartitionLabel";
-      this.dataPartitionLabel.Size = new System.Drawing.Size(73, 13);
-      this.dataPartitionLabel.TabIndex = 0;
-      this.dataPartitionLabel.Text = "Data partition:";
-      // 
-      // numericVarReplacementLabel
-      // 
-      this.numericVarReplacementLabel.AutoSize = true;
-      this.numericVarReplacementLabel.Location = new System.Drawing.Point(3, 33);
-      this.numericVarReplacementLabel.Name = "numericVarReplacementLabel";
-      this.numericVarReplacementLabel.Size = new System.Drawing.Size(173, 13);
-      this.numericVarReplacementLabel.TabIndex = 2;
-      this.numericVarReplacementLabel.Text = "Replacement for numeric variables:";
+      // ascendingCheckBox
+      // 
+      this.ascendingCheckBox.AutoSize = true;
+      this.ascendingCheckBox.CheckAlign = System.Drawing.ContentAlignment.MiddleRight;
+      this.ascendingCheckBox.Location = new System.Drawing.Point(452, 32);
+      this.ascendingCheckBox.Name = "ascendingCheckBox";
+      this.ascendingCheckBox.Size = new System.Drawing.Size(76, 17);
+      this.ascendingCheckBox.TabIndex = 7;
+      this.ascendingCheckBox.Text = "Ascending";
+      this.ascendingCheckBox.UseVisualStyleBackColor = true;
+      this.ascendingCheckBox.CheckedChanged += new System.EventHandler(this.ascendingCheckBox_CheckedChanged);
+      // 
+      // sortByLabel
+      // 
+      this.sortByLabel.AutoSize = true;
+      this.sortByLabel.Location = new System.Drawing.Point(324, 6);
+      this.sortByLabel.Name = "sortByLabel";
+      this.sortByLabel.Size = new System.Drawing.Size(77, 13);
+      this.sortByLabel.TabIndex = 4;
+      this.sortByLabel.Text = "Sorting criteria:";
+      // 
+      // sortByComboBox
+      // 
+      this.sortByComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.sortByComboBox.FormattingEnabled = true;
+      this.sortByComboBox.Items.AddRange(new object[] {
+            HeuristicLab.Problems.DataAnalysis.Views.ClassificationSolutionVariableImpactsView.SortingCriteria.ImpactValue,
+            HeuristicLab.Problems.DataAnalysis.Views.ClassificationSolutionVariableImpactsView.SortingCriteria.Occurrence,
+            HeuristicLab.Problems.DataAnalysis.Views.ClassificationSolutionVariableImpactsView.SortingCriteria.VariableName});
+      this.sortByComboBox.Location = new System.Drawing.Point(407, 3);
+      this.sortByComboBox.Name = "sortByComboBox";
+      this.sortByComboBox.Size = new System.Drawing.Size(121, 21);
+      this.sortByComboBox.TabIndex = 5;
+      this.sortByComboBox.SelectedIndexChanged += new System.EventHandler(this.sortByComboBox_SelectedIndexChanged);
+      // 
+      // factorVarReplComboBox
+      // 
+      this.factorVarReplComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.factorVarReplComboBox.FormattingEnabled = true;
+      this.factorVarReplComboBox.Items.AddRange(new object[] {
+            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.FactorReplacementMethodEnum.Best,
+            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.FactorReplacementMethodEnum.Mode,
+            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.FactorReplacementMethodEnum.Shuffle});
+      this.factorVarReplComboBox.Location = new System.Drawing.Point(197, 57);
+      this.factorVarReplComboBox.Name = "factorVarReplComboBox";
+      this.factorVarReplComboBox.Size = new System.Drawing.Size(121, 21);
+      this.factorVarReplComboBox.TabIndex = 1;
+      this.factorVarReplComboBox.SelectedIndexChanged += new System.EventHandler(this.replacementComboBox_SelectedIndexChanged);
+      // 
+      // factorVarReplacementLabel
+      // 
+      this.factorVarReplacementLabel.AutoSize = true;
+      this.factorVarReplacementLabel.Location = new System.Drawing.Point(3, 60);
+      this.factorVarReplacementLabel.Name = "factorVarReplacementLabel";
+      this.factorVarReplacementLabel.Size = new System.Drawing.Size(188, 13);
+      this.factorVarReplacementLabel.TabIndex = 0;
+      this.factorVarReplacementLabel.Text = "Replacement for categorical variables:";
       // 
       // replacementComboBox
@@ -117 +131 @@
       this.replacementComboBox.SelectedIndexChanged += new System.EventHandler(this.replacementComboBox_SelectedIndexChanged);
       // 
-      // factorVarReplacementLabel
-      // 
-      this.factorVarReplacementLabel.AutoSize = true;
-      this.factorVarReplacementLabel.Location = new System.Drawing.Point(3, 60);
-      this.factorVarReplacementLabel.Name = "factorVarReplacementLabel";
-      this.factorVarReplacementLabel.Size = new System.Drawing.Size(188, 13);
-      this.factorVarReplacementLabel.TabIndex = 0;
-      this.factorVarReplacementLabel.Text = "Replacement for categorical variables:";
-      // 
-      // factorVarReplComboBox
-      // 
-      this.factorVarReplComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
-      this.factorVarReplComboBox.FormattingEnabled = true;
-      this.factorVarReplComboBox.Items.AddRange(new object[] {
-            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.FactorReplacementMethodEnum.Best,
-            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.FactorReplacementMethodEnum.Mode,
-            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.FactorReplacementMethodEnum.Shuffle});
-      this.factorVarReplComboBox.Location = new System.Drawing.Point(197, 57);
-      this.factorVarReplComboBox.Name = "factorVarReplComboBox";
-      this.factorVarReplComboBox.Size = new System.Drawing.Size(121, 21);
-      this.factorVarReplComboBox.TabIndex = 1;
-      this.factorVarReplComboBox.SelectedIndexChanged += new System.EventHandler(this.replacementComboBox_SelectedIndexChanged);
-      // 
-      // ascendingCheckBox
-      // 
-      this.ascendingCheckBox.AutoSize = true;
-      this.ascendingCheckBox.Location = new System.Drawing.Point(534, 6);
-      this.ascendingCheckBox.Name = "ascendingCheckBox";
-      this.ascendingCheckBox.Size = new System.Drawing.Size(76, 17);
-      this.ascendingCheckBox.TabIndex = 10;
-      this.ascendingCheckBox.Text = "Ascending";
-      this.ascendingCheckBox.UseVisualStyleBackColor = true;
-      this.ascendingCheckBox.CheckedChanged += new System.EventHandler(this.ascendingCheckBox_CheckedChanged);
-      // 
-      // sortByLabel
-      // 
-      this.sortByLabel.AutoSize = true;
-      this.sortByLabel.Location = new System.Drawing.Point(324, 6);
-      this.sortByLabel.Name = "sortByLabel";
-      this.sortByLabel.Size = new System.Drawing.Size(77, 13);
-      this.sortByLabel.TabIndex = 8;
-      this.sortByLabel.Text = "Sorting criteria:";
-      // 
-      // sortByComboBox
-      // 
-      this.sortByComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
-      this.sortByComboBox.FormattingEnabled = true;
-      this.sortByComboBox.Location = new System.Drawing.Point(407, 3);
-      this.sortByComboBox.Name = "sortByComboBox";
-      this.sortByComboBox.Size = new System.Drawing.Size(121, 21);
-      this.sortByComboBox.TabIndex = 9;
-      this.sortByComboBox.SelectedIndexChanged += new System.EventHandler(this.sortByComboBox_SelectedIndexChanged);
+      // numericVarReplacementLabel
+      // 
+      this.numericVarReplacementLabel.AutoSize = true;
+      this.numericVarReplacementLabel.Location = new System.Drawing.Point(3, 33);
+      this.numericVarReplacementLabel.Name = "numericVarReplacementLabel";
+      this.numericVarReplacementLabel.Size = new System.Drawing.Size(173, 13);
+      this.numericVarReplacementLabel.TabIndex = 2;
+      this.numericVarReplacementLabel.Text = "Replacement for numeric variables:";
+      // 
+      // dataPartitionLabel
+      // 
+      this.dataPartitionLabel.AutoSize = true;
+      this.dataPartitionLabel.Location = new System.Drawing.Point(3, 6);
+      this.dataPartitionLabel.Name = "dataPartitionLabel";
+      this.dataPartitionLabel.Size = new System.Drawing.Size(73, 13);
+      this.dataPartitionLabel.TabIndex = 0;
+      this.dataPartitionLabel.Text = "Data partition:";
+      // 
+      // dataPartitionComboBox
+      // 
+      this.dataPartitionComboBox.DropDownStyle = System.Windows.Forms.ComboBoxStyle.DropDownList;
+      this.dataPartitionComboBox.FormattingEnabled = true;
+      this.dataPartitionComboBox.Items.AddRange(new object[] {
+            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum.Training,
+            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum.Test,
+            HeuristicLab.Problems.DataAnalysis.ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum.All});
+      this.dataPartitionComboBox.Location = new System.Drawing.Point(197, 3);
+      this.dataPartitionComboBox.Name = "dataPartitionComboBox";
+      this.dataPartitionComboBox.Size = new System.Drawing.Size(121, 21);
+      this.dataPartitionComboBox.TabIndex = 1;
+      this.dataPartitionComboBox.SelectedIndexChanged += new System.EventHandler(this.dataPartitionComboBox_SelectedIndexChanged);
+      // 
+      // variableImpactsArrayView
+      // 
+      this.variableImpactsArrayView.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
+            | System.Windows.Forms.AnchorStyles.Left) 
+            | System.Windows.Forms.AnchorStyles.Right)));
+      this.variableImpactsArrayView.Caption = "StringConvertibleArray View";
+      this.variableImpactsArrayView.Content = null;
+      this.variableImpactsArrayView.Location = new System.Drawing.Point(3, 84);
+      this.variableImpactsArrayView.Name = "variableImpactsArrayView";
+      this.variableImpactsArrayView.ReadOnly = true;
+      this.variableImpactsArrayView.Size = new System.Drawing.Size(706, 278);
+      this.variableImpactsArrayView.TabIndex = 2;
       // 
       // ClassificationSolutionVariableImpactsView
@@ -183 +189 @@
       this.Controls.Add(this.dataPartitionLabel);
       this.Controls.Add(this.dataPartitionComboBox);
-      this.Controls.Add(this.variableImactsArrayView);
+      this.Controls.Add(this.variableImpactsArrayView);
       this.Name = "ClassificationSolutionVariableImpactsView";
-      this.Size = new System.Drawing.Size(668, 365);
+      this.Size = new System.Drawing.Size(712, 365);
       this.VisibleChanged += new System.EventHandler(this.ClassificationSolutionVariableImpactsView_VisibleChanged);
       this.ResumeLayout(false);
@@ -194 +200 @@
     #endregion
 
-    private Data.Views.StringConvertibleArrayView variableImactsArrayView;
+    private Data.Views.StringConvertibleArrayView variableImpactsArrayView;
     private System.Windows.Forms.ComboBox dataPartitionComboBox;
     private System.Windows.Forms.Label dataPartitionLabel;
@@ -201 +207 @@
     private System.Windows.Forms.Label factorVarReplacementLabel;
     private System.Windows.Forms.ComboBox factorVarReplComboBox;
-    private System.Windows.Forms.CheckBox ascendingCheckBox;
     private System.Windows.Forms.Label sortByLabel;
     private System.Windows.Forms.ComboBox sortByComboBox;
-    private System.ComponentModel.BackgroundWorker backgroundWorker;
+    private System.Windows.Forms.CheckBox ascendingCheckBox;
   }
 }

trunk/HeuristicLab.Problems.DataAnalysis.Views/3.4/Classification/ClassificationSolutionVariableImpactsView.cs

@@ -33 +33 @@
   [Content(typeof(IClassificationSolution))]
   public partial class ClassificationSolutionVariableImpactsView : DataAnalysisSolutionEvaluationView {
-    #region Nested Types
     private enum SortingCriteria {
       ImpactValue,
@@ -39 +38 @@
       VariableName
     }
-    #endregion
-
-    #region Fields
-    private Dictionary<string, double> rawVariableImpacts = new Dictionary<string, double>();
-    private Thread thread;
-    #endregion
-
-    #region Getter/Setter
+    private CancellationTokenSource cancellationToken = new CancellationTokenSource();
+    private List<Tuple<string, double>> rawVariableImpacts = new List<Tuple<string, double>>();
+
     public new IClassificationSolution Content {
       get { return (IClassificationSolution)base.Content; }
@@ -53 +47 @@
       }
     }
-    #endregion
-
-    #region Ctor
+
     public ClassificationSolutionVariableImpactsView()
       : base() {
       InitializeComponent();
 
-      this.sortByComboBox.Items.AddRange(Enum.GetValues(typeof(SortingCriteria)).Cast<object>().ToArray());
-      this.sortByComboBox.SelectedItem = SortingCriteria.ImpactValue;
-
       //Set the default values
       this.dataPartitionComboBox.SelectedIndex = 0;
-      this.replacementComboBox.SelectedIndex = 0;
+      this.replacementComboBox.SelectedIndex = 3;
       this.factorVarReplComboBox.SelectedIndex = 0;
-    }
-    #endregion
-
-    #region Events
+      this.sortByComboBox.SelectedItem = SortingCriteria.ImpactValue;
+    }
+
     protected override void RegisterContentEvents() {
       base.RegisterContentEvents();
@@ -76 +64 @@
       Content.ProblemDataChanged += new EventHandler(Content_ProblemDataChanged);
     }
-
     protected override void DeregisterContentEvents() {
       base.DeregisterContentEvents();
@@ -86 +73 @@
       OnContentChanged();
     }
-
     protected virtual void Content_ModelChanged(object sender, EventArgs e) {
       OnContentChanged();
     }
-
     protected override void OnContentChanged() {
       base.OnContentChanged();
+      rawVariableImpacts.Clear();
+
       if (Content == null) {
-        variableImactsArrayView.Content = null;
+        variableImpactsArrayView.Content = null;
       } else {
         UpdateVariableImpact();
       }
     }
-
     private void ClassificationSolutionVariableImpactsView_VisibleChanged(object sender, EventArgs e) {
-      if (thread == null) { return; }
-
-      if (thread.IsAlive) { thread.Abort(); }
-      thread = null;
-    }
-
+      cancellationToken.Cancel();
+    }
 
     private void dataPartitionComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      rawVariableImpacts.Clear();
       UpdateVariableImpact();
     }
-
     private void replacementComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      rawVariableImpacts.Clear();
       UpdateVariableImpact();
     }
-
     private void sortByComboBox_SelectedIndexChanged(object sender, EventArgs e) {
       //Update the default ordering (asc,desc), but remove the eventHandler beforehand (otherwise the data would be ordered twice)
       ascendingCheckBox.CheckedChanged -= ascendingCheckBox_CheckedChanged;
-      switch ((SortingCriteria)sortByComboBox.SelectedItem) {
-        case SortingCriteria.ImpactValue:
-          ascendingCheckBox.Checked = false;
-          break;
-        case SortingCriteria.Occurrence:
-          ascendingCheckBox.Checked = true;
-          break;
-        case SortingCriteria.VariableName:
-          ascendingCheckBox.Checked = true;
-          break;
-        default:
-          throw new NotImplementedException("Ordering for selected SortingCriteria not implemented");
-      }
+      ascendingCheckBox.Checked = (SortingCriteria)sortByComboBox.SelectedItem != SortingCriteria.ImpactValue;
       ascendingCheckBox.CheckedChanged += ascendingCheckBox_CheckedChanged;
 
-      UpdateDataOrdering();
-    }
-
+      UpdateOrdering();
+    }
     private void ascendingCheckBox_CheckedChanged(object sender, EventArgs e) {
-      UpdateDataOrdering();
-    }
-
-    #endregion
-
-    #region Helper Methods   
-    private void UpdateVariableImpact() {
+      UpdateOrdering();
+    }
+
+    private async void UpdateVariableImpact() {
+      IProgress progress;
+
       //Check if the selection is valid
       if (Content == null) { return; }
@@ -157 +125 @@
       var dataPartition = (ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum)dataPartitionComboBox.SelectedItem;
 
-      variableImactsArrayView.Caption = Content.Name + " Variable Impacts";
-
-      mainForm.AddOperationProgressToView(this, "Calculating variable impacts for " + Content.Name);
-
-      Task.Factory.StartNew(() => {
-        thread = Thread.CurrentThread;
-        //Remember the original ordering of the variables
-        var impacts = ClassificationSolutionVariableImpactsCalculator.CalculateImpacts(Content, dataPartition, replMethod, factorReplMethod);
+      variableImpactsArrayView.Caption = Content.Name + " Variable Impacts";
+      progress = mainForm.AddOperationProgressToView(this, "Calculating variable impacts for " + Content.Name);
+      progress.ProgressValue = 0;
+
+      cancellationToken = new CancellationTokenSource();
+
+      try {
         var problemData = Content.ProblemData;
         var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(Content.Model.VariablesUsedForPrediction));
-        var originalVariableOrdering = problemData.Dataset.VariableNames.Where(v => inputvariables.Contains(v)).Where(problemData.Dataset.VariableHasType<double>).ToList();
-
-        rawVariableImpacts.Clear();
-        originalVariableOrdering.ForEach(v => rawVariableImpacts.Add(v, impacts.First(vv => vv.Item1 == v).Item2));
-      }).ContinueWith((o) => {
-        UpdateDataOrdering();
-        mainForm.RemoveOperationProgressFromView(this);
-        thread = null;
-      }, TaskScheduler.FromCurrentSynchronizationContext());
+        //Remember the original ordering of the variables
+        var originalVariableOrdering = problemData.Dataset.VariableNames
+          .Where(v => inputvariables.Contains(v))
+          .Where(v => problemData.Dataset.VariableHasType<double>(v) || problemData.Dataset.VariableHasType<string>(v))
+          .ToList();
+
+        List<Tuple<string, double>> impacts = null;
+        await Task.Run(() => { impacts = CalculateVariableImpacts(originalVariableOrdering, Content.Model, problemData, Content.EstimatedClassValues, dataPartition, replMethod, factorReplMethod, cancellationToken.Token, progress); });
+        if (impacts == null) { return; }
+
+        rawVariableImpacts.AddRange(impacts);
+        UpdateOrdering();
+      }
+      finally {
+        ((MainForm.WindowsForms.MainForm)MainFormManager.MainForm).RemoveOperationProgressFromView(this);
+      }
+    }
+    private List<Tuple<string, double>> CalculateVariableImpacts(List<string> originalVariableOrdering,
+      IClassificationModel model,
+      IClassificationProblemData problemData,
+      IEnumerable<double> estimatedClassValues,
+      ClassificationSolutionVariableImpactsCalculator.DataPartitionEnum dataPartition,
+      ClassificationSolutionVariableImpactsCalculator.ReplacementMethodEnum replMethod,
+      ClassificationSolutionVariableImpactsCalculator.FactorReplacementMethodEnum factorReplMethod,
+      CancellationToken token,
+      IProgress progress) {
+      List<Tuple<string, double>> impacts = new List<Tuple<string, double>>();
+      int count = originalVariableOrdering.Count;
+      int i = 0;
+      var modifiableDataset = ((Dataset)(problemData.Dataset).Clone()).ToModifiable();
+      IEnumerable<int> rows = ClassificationSolutionVariableImpactsCalculator.GetPartitionRows(dataPartition, problemData);
+
+      //Calculate original quality-values (via calculator, default is R²)
+      IEnumerable<double> targetValuesPartition = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      IEnumerable<double> estimatedClassValuesPartition = Content.GetEstimatedClassValues(rows);
+
+      var originalCalculatorValue = ClassificationSolutionVariableImpactsCalculator.CalculateQuality(targetValuesPartition, estimatedClassValuesPartition);
+      var clonedModel = (IClassificationModel)model.Clone();
+      foreach (var variableName in originalVariableOrdering) {
+        if (cancellationToken.Token.IsCancellationRequested) { return null; }
+        progress.ProgressValue = (double)++i / count;
+        progress.Status = string.Format("Calculating impact for variable {0} ({1} of {2})", variableName, i, count);
+
+        double impact = 0;
+        //If the variable isn't used for prediction, it has zero impact.
+        if (model.VariablesUsedForPrediction.Contains(variableName)) {
+          impact = ClassificationSolutionVariableImpactsCalculator.CalculateImpact(variableName, clonedModel, problemData, modifiableDataset, rows, replMethod, factorReplMethod, targetValuesPartition, originalCalculatorValue);
+        }
+        impacts.Add(new Tuple<string, double>(variableName, impact));
+      }
+
+      return impacts;
     }
 
     /// <summary>
-    /// Updates the <see cref="variableImactsArrayView"/> according to the selected ordering <see cref="ascendingCheckBox"/> of the selected Column <see cref="sortByComboBox"/>
+    /// Updates the <see cref="variableImpactsArrayView"/> according to the selected ordering <see cref="ascendingCheckBox"/> of the selected Column <see cref="sortByComboBox"/>
     /// The default is "Descending" by "VariableImpact" (as in previous versions)
     /// </summary>
-    private void UpdateDataOrdering() {
+    private void UpdateOrdering() {
       //Check if valid sortingCriteria is selected and data exists
       if (sortByComboBox.SelectedIndex == -1) { return; }
@@ -191 +201 @@
       bool ascending = ascendingCheckBox.Checked;
 
-      IEnumerable<KeyValuePair<string, double>> orderedEntries = null;
+      IEnumerable<Tuple<string, double>> orderedEntries = null;
 
       //Sort accordingly
       switch (selectedItem) {
         case SortingCriteria.ImpactValue:
-          orderedEntries = rawVariableImpacts.OrderBy(v => v.Value);
+          orderedEntries = rawVariableImpacts.OrderBy(v => v.Item2);
           break;
         case SortingCriteria.Occurrence:
@@ -202 +212 @@
           break;
         case SortingCriteria.VariableName:
-          orderedEntries = rawVariableImpacts.OrderBy(v => v.Key, new NaturalStringComparer());
+          orderedEntries = rawVariableImpacts.OrderBy(v => v.Item1, new NaturalStringComparer());
           break;
         default:
@@ -211 +221 @@
 
       //Write the data back
-      var impactArray = new DoubleArray(orderedEntries.Select(i => i.Value).ToArray()) {
-        ElementNames = orderedEntries.Select(i => i.Key)
+      var impactArray = new DoubleArray(orderedEntries.Select(i => i.Item2).ToArray()) {
+        ElementNames = orderedEntries.Select(i => i.Item1)
       };
 
       //Could be, if the View was closed
-      if (!variableImactsArrayView.IsDisposed) {
-        variableImactsArrayView.Content = (DoubleArray)impactArray.AsReadOnly();
-      }
-    }
-    #endregion  
+      if (!variableImpactsArrayView.IsDisposed) {
+        variableImpactsArrayView.Content = (DoubleArray)impactArray.AsReadOnly();
+      }
+    }
   }
 }

trunk/HeuristicLab.Problems.DataAnalysis.Views/3.4/Regression/RegressionSolutionVariableImpactsView.Designer.cs

@@ -55 +55 @@
       this.dataPartitionLabel = new System.Windows.Forms.Label();
       this.dataPartitionComboBox = new System.Windows.Forms.ComboBox();
-      this.variableImactsArrayView = new HeuristicLab.Data.Views.StringConvertibleArrayView();
+      this.variableImpactsArrayView = new HeuristicLab.Data.Views.StringConvertibleArrayView();
       this.SuspendLayout();
       // 
@@ -61 +61 @@
       // 
       this.ascendingCheckBox.AutoSize = true;
-      this.ascendingCheckBox.Location = new System.Drawing.Point(534, 6);
+      this.ascendingCheckBox.CheckAlign = System.Drawing.ContentAlignment.MiddleRight;
+      this.ascendingCheckBox.Location = new System.Drawing.Point(452, 32);
       this.ascendingCheckBox.Name = "ascendingCheckBox";
       this.ascendingCheckBox.Size = new System.Drawing.Size(76, 17);
@@ -162 +163 @@
       this.dataPartitionComboBox.SelectedIndexChanged += new System.EventHandler(this.dataPartitionComboBox_SelectedIndexChanged);
       // 
-      // variableImactsArrayView
-      // 
-      this.variableImactsArrayView.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
-            | System.Windows.Forms.AnchorStyles.Left)
+      // variableImpactsArrayView
+      // 
+      this.variableImpactsArrayView.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
+            | System.Windows.Forms.AnchorStyles.Left) 
             | System.Windows.Forms.AnchorStyles.Right)));
-      this.variableImactsArrayView.Caption = "StringConvertibleArray View";
-      this.variableImactsArrayView.Content = null;
-      this.variableImactsArrayView.Location = new System.Drawing.Point(3, 84);
-      this.variableImactsArrayView.Name = "variableImactsArrayView";
-      this.variableImactsArrayView.ReadOnly = true;
-      this.variableImactsArrayView.Size = new System.Drawing.Size(706, 278);
-      this.variableImactsArrayView.TabIndex = 2;
+      this.variableImpactsArrayView.Caption = "StringConvertibleArray View";
+      this.variableImpactsArrayView.Content = null;
+      this.variableImpactsArrayView.Location = new System.Drawing.Point(3, 84);
+      this.variableImpactsArrayView.Name = "variableImpactsArrayView";
+      this.variableImpactsArrayView.ReadOnly = true;
+      this.variableImpactsArrayView.Size = new System.Drawing.Size(706, 278);
+      this.variableImpactsArrayView.TabIndex = 2;
       // 
       // RegressionSolutionVariableImpactsView
@@ -188 +189 @@
       this.Controls.Add(this.dataPartitionLabel);
       this.Controls.Add(this.dataPartitionComboBox);
-      this.Controls.Add(this.variableImactsArrayView);
+      this.Controls.Add(this.variableImpactsArrayView);
       this.Name = "RegressionSolutionVariableImpactsView";
       this.Size = new System.Drawing.Size(712, 365);
@@ -199 +200 @@
     #endregion
 
-    private Data.Views.StringConvertibleArrayView variableImactsArrayView;
+    private Data.Views.StringConvertibleArrayView variableImpactsArrayView;
     private System.Windows.Forms.ComboBox dataPartitionComboBox;
     private System.Windows.Forms.Label dataPartitionLabel;

trunk/HeuristicLab.Problems.DataAnalysis.Views/3.4/Regression/RegressionSolutionVariableImpactsView.cs

@@ -33 +33 @@
   [Content(typeof(IRegressionSolution))]
   public partial class RegressionSolutionVariableImpactsView : DataAnalysisSolutionEvaluationView {
-    private CancellationTokenSource cancellationToken = new CancellationTokenSource();
     private enum SortingCriteria {
       ImpactValue,
@@ -39 +38 @@
       VariableName
     }
+    private CancellationTokenSource cancellationToken = new CancellationTokenSource();
     private List<Tuple<string, double>> rawVariableImpacts = new List<Tuple<string, double>>();
 
@@ -64 +64 @@
       Content.ProblemDataChanged += new EventHandler(Content_ProblemDataChanged);
     }
-
     protected override void DeregisterContentEvents() {
       base.DeregisterContentEvents();
@@ -74 +73 @@
       OnContentChanged();
     }
-
     protected virtual void Content_ModelChanged(object sender, EventArgs e) {
       OnContentChanged();
     }
-
     protected override void OnContentChanged() {
       base.OnContentChanged();
+      rawVariableImpacts.Clear();
+
       if (Content == null) {
-        variableImactsArrayView.Content = null;
+        variableImpactsArrayView.Content = null;
       } else {
         UpdateVariableImpact();
       }
     }
-
     private void RegressionSolutionVariableImpactsView_VisibleChanged(object sender, EventArgs e) {
       cancellationToken.Cancel();
     }
 
-
     private void dataPartitionComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      rawVariableImpacts.Clear();
       UpdateVariableImpact();
     }
-
     private void replacementComboBox_SelectedIndexChanged(object sender, EventArgs e) {
+      rawVariableImpacts.Clear();
       UpdateVariableImpact();
     }
-
     private void sortByComboBox_SelectedIndexChanged(object sender, EventArgs e) {
       //Update the default ordering (asc,desc), but remove the eventHandler beforehand (otherwise the data would be ordered twice)
@@ -109 +106 @@
       UpdateOrdering();
     }
-
     private void ascendingCheckBox_CheckedChanged(object sender, EventArgs e) {
       UpdateOrdering();
     }
-
 
     private async void UpdateVariableImpact() {
@@ -130 +125 @@
       var dataPartition = (RegressionSolutionVariableImpactsCalculator.DataPartitionEnum)dataPartitionComboBox.SelectedItem;
 
-      variableImactsArrayView.Caption = Content.Name + " Variable Impacts";
+      variableImpactsArrayView.Caption = Content.Name + " Variable Impacts";
       progress = mainForm.AddOperationProgressToView(this, "Calculating variable impacts for " + Content.Name);
       progress.ProgressValue = 0;
 
       cancellationToken = new CancellationTokenSource();
-      //Remember the original ordering of the variables
+
       try {
-        var impacts = await Task.Run(() => RegressionSolutionVariableImpactsCalculator.CalculateImpacts(Content, dataPartition, replMethod, factorReplMethod,
-          (i, s) => {
-            progress.ProgressValue = i;
-            progress.Status = s;
-            return cancellationToken.Token.IsCancellationRequested;
-          }), cancellationToken.Token);
-
-        if (cancellationToken.Token.IsCancellationRequested) { return; }
         var problemData = Content.ProblemData;
         var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(Content.Model.VariablesUsedForPrediction));
+        //Remember the original ordering of the variables
         var originalVariableOrdering = problemData.Dataset.VariableNames
           .Where(v => inputvariables.Contains(v))
@@ -152 +140 @@
           .ToList();
 
-        rawVariableImpacts.Clear();
-        originalVariableOrdering.ForEach(v => rawVariableImpacts.Add(new Tuple<string, double>(v, impacts.First(vv => vv.Item1 == v).Item2)));
+        List<Tuple<string, double>> impacts = null;
+        await Task.Run(() => { impacts = CalculateVariableImpacts(originalVariableOrdering, Content.Model, problemData, Content.EstimatedValues, dataPartition, replMethod, factorReplMethod, cancellationToken.Token, progress); });
+        if (impacts == null) { return; }
+
+        rawVariableImpacts.AddRange(impacts);
         UpdateOrdering();
-      } finally {
+      }
+      finally {
         ((MainForm.WindowsForms.MainForm)MainFormManager.MainForm).RemoveOperationProgressFromView(this);
       }
     }
+    private List<Tuple<string, double>> CalculateVariableImpacts(List<string> originalVariableOrdering,
+      IRegressionModel model,
+      IRegressionProblemData problemData,
+      IEnumerable<double> estimatedValues,
+      RegressionSolutionVariableImpactsCalculator.DataPartitionEnum dataPartition,
+      RegressionSolutionVariableImpactsCalculator.ReplacementMethodEnum replMethod,
+      RegressionSolutionVariableImpactsCalculator.FactorReplacementMethodEnum factorReplMethod,
+      CancellationToken token,
+      IProgress progress) {
+      List<Tuple<string, double>> impacts = new List<Tuple<string, double>>();
+      int count = originalVariableOrdering.Count;
+      int i = 0;
+      var modifiableDataset = ((Dataset)(problemData.Dataset).Clone()).ToModifiable();
+      IEnumerable<int> rows = RegressionSolutionVariableImpactsCalculator.GetPartitionRows(dataPartition, problemData);
+
+      //Calculate original quality-values (via calculator, default is R²)
+      IEnumerable<double> targetValuesPartition = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      IEnumerable<double> estimatedValuesPartition = Content.GetEstimatedValues(rows);
+
+      var originalCalculatorValue = RegressionSolutionVariableImpactsCalculator.CalculateQuality(targetValuesPartition, estimatedValuesPartition);
+
+      foreach (var variableName in originalVariableOrdering) {
+        if (cancellationToken.Token.IsCancellationRequested) { return null; }
+        progress.ProgressValue = (double)++i / count;
+        progress.Status = string.Format("Calculating impact for variable {0} ({1} of {2})", variableName, i, count);
+
+        double impact = 0;
+        //If the variable isn't used for prediction, it has zero impact.
+        if (model.VariablesUsedForPrediction.Contains(variableName)) {
+          impact = RegressionSolutionVariableImpactsCalculator.CalculateImpact(variableName, model, problemData, modifiableDataset, rows, replMethod, factorReplMethod, targetValuesPartition, originalCalculatorValue);
+        }
+        impacts.Add(new Tuple<string, double>(variableName, impact));
+      }
+
+      return impacts;
+    }
 
     /// <summary>
-    /// Updates the <see cref="variableImactsArrayView"/> according to the selected ordering <see cref="ascendingCheckBox"/> of the selected Column <see cref="sortByComboBox"/>
+    /// Updates the <see cref="variableImpactsArrayView"/> according to the selected ordering <see cref="ascendingCheckBox"/> of the selected Column <see cref="sortByComboBox"/>
     /// The default is "Descending" by "VariableImpact" (as in previous versions)
     /// </summary>
@@ -198 +226 @@
 
       //Could be, if the View was closed
-      if (!variableImactsArrayView.IsDisposed) {
-        variableImactsArrayView.Content = (DoubleArray)impactArray.AsReadOnly();
+      if (!variableImpactsArrayView.IsDisposed) {
+        variableImpactsArrayView.Content = (DoubleArray)impactArray.AsReadOnly();
       }
     }

trunk/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionVariableImpactsCalculator.cs

@@ -23 +23 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
@@ -36 +37 @@
   [Item("ClassificationSolution Impacts Calculator", "Calculation of the impacts of input variables for any classification solution")]
   public sealed class ClassificationSolutionVariableImpactsCalculator : ParameterizedNamedItem {
+    #region Parameters/Properties
     public enum ReplacementMethodEnum {
       Median,
@@ -54 +56 @@
 
     private const string ReplacementParameterName = "Replacement Method";
+    private const string FactorReplacementParameterName = "Factor Replacement Method";
     private const string DataPartitionParameterName = "DataPartition";
 
     public IFixedValueParameter<EnumValue<ReplacementMethodEnum>> ReplacementParameter {
       get { return (IFixedValueParameter<EnumValue<ReplacementMethodEnum>>)Parameters[ReplacementParameterName]; }
+    }
+    public IFixedValueParameter<EnumValue<FactorReplacementMethodEnum>> FactorReplacementParameter {
+      get { return (IFixedValueParameter<EnumValue<FactorReplacementMethodEnum>>)Parameters[FactorReplacementParameterName]; }
     }
     public IFixedValueParameter<EnumValue<DataPartitionEnum>> DataPartitionParameter {
@@ -67 +73 @@
       set { ReplacementParameter.Value.Value = value; }
     }
+    public FactorReplacementMethodEnum FactorReplacementMethod {
+      get { return FactorReplacementParameter.Value.Value; }
+      set { FactorReplacementParameter.Value.Value = value; }
+    }
     public DataPartitionEnum DataPartition {
       get { return DataPartitionParameter.Value.Value; }
       set { DataPartitionParameter.Value.Value = value; }
     }
-
-
+    #endregion
+
+    #region Ctor/Cloner
     [StorableConstructor]
     private ClassificationSolutionVariableImpactsCalculator(bool deserializing) : base(deserializing) { }
     private ClassificationSolutionVariableImpactsCalculator(ClassificationSolutionVariableImpactsCalculator original, Cloner cloner)
       : base(original, cloner) { }
+    public ClassificationSolutionVariableImpactsCalculator()
+      : base() {
+      Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum.Shuffle)));
+      Parameters.Add(new FixedValueParameter<EnumValue<FactorReplacementMethodEnum>>(FactorReplacementParameterName, "The replacement method for factor variables during impact calculation.", new EnumValue<FactorReplacementMethodEnum>(FactorReplacementMethodEnum.Best)));
+      Parameters.Add(new FixedValueParameter<EnumValue<DataPartitionEnum>>(DataPartitionParameterName, "The data partition on which the impacts are calculated.", new EnumValue<DataPartitionEnum>(DataPartitionEnum.Training)));
+    }
+
     public override IDeepCloneable Clone(Cloner cloner) {
       return new ClassificationSolutionVariableImpactsCalculator(this, cloner);
     }
-
-    public ClassificationSolutionVariableImpactsCalculator()
-      : base() {
-      Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum.Median)));
-      Parameters.Add(new FixedValueParameter<EnumValue<DataPartitionEnum>>(DataPartitionParameterName, "The data partition on which the impacts are calculated.", new EnumValue<DataPartitionEnum>(DataPartitionEnum.Training)));
-    }
+    #endregion
 
     //mkommend: annoying name clash with static method, open to better naming suggestions
     public IEnumerable<Tuple<string, double>> Calculate(IClassificationSolution solution) {
-      return CalculateImpacts(solution, DataPartition, ReplacementMethod);
+      return CalculateImpacts(solution, ReplacementMethod, FactorReplacementMethod, DataPartition);
     }
 
     public static IEnumerable<Tuple<string, double>> CalculateImpacts(
       IClassificationSolution solution,
-      DataPartitionEnum data = DataPartitionEnum.Training,
-      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
+      DataPartitionEnum dataPartition = DataPartitionEnum.Training) {
+
+      IEnumerable<int> rows = GetPartitionRows(dataPartition, solution.ProblemData);
+      IEnumerable<double> estimatedClassValues = solution.GetEstimatedClassValues(rows);
+      var model = (IClassificationModel)solution.Model.Clone(); //mkommend: clone of model is necessary, because the thresholds for IDiscriminantClassificationModels are updated
+
+      return CalculateImpacts(model, solution.ProblemData, estimatedClassValues, rows, replacementMethod, factorReplacementMethod);
+    }
+
+    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
+     IClassificationModel model,
+     IClassificationProblemData problemData,
+     IEnumerable<double> estimatedClassValues,
+     IEnumerable<int> rows,
+     ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
+     FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
+
+      //fholzing: try and catch in case a different dataset is loaded, otherwise statement is neglectable
+      var missingVariables = model.VariablesUsedForPrediction.Except(problemData.Dataset.VariableNames);
+      if (missingVariables.Any()) {
+        throw new InvalidOperationException(string.Format("Can not calculate variable impacts, because the model uses inputs missing in the dataset ({0})", string.Join(", ", missingVariables)));
+      }
+      IEnumerable<double> targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      var originalQuality = CalculateQuality(targetValues, estimatedClassValues);
+
+      var impacts = new Dictionary<string, double>();
+      var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(model.VariablesUsedForPrediction));
+      var modifiableDataset = ((Dataset)(problemData.Dataset).Clone()).ToModifiable();
+
+      foreach (var inputVariable in inputvariables) {
+        impacts[inputVariable] = CalculateImpact(inputVariable, model, problemData, modifiableDataset, rows, replacementMethod, factorReplacementMethod, targetValues, originalQuality);
+      }
+
+      return impacts.Select(i => Tuple.Create(i.Key, i.Value));
+    }
+
+    public static double CalculateImpact(string variableName,
+      IClassificationModel model,
+      IClassificationProblemData problemData,
+      ModifiableDataset modifiableDataset,
+      IEnumerable<int> rows,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
+      IEnumerable<double> targetValues = null,
+      double quality = double.NaN) {
+
+      if (!model.VariablesUsedForPrediction.Contains(variableName)) { return 0.0; }
+      if (!problemData.Dataset.VariableNames.Contains(variableName)) {
+        throw new InvalidOperationException(string.Format("Can not calculate variable impact, because the model uses inputs missing in the dataset ({0})", variableName));
+      }
+
+      if (targetValues == null) {
+        targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      }
+      if (quality == double.NaN) {
+        quality = CalculateQuality(model.GetEstimatedClassValues(modifiableDataset, rows), targetValues);
+      }
+
+      IList originalValues = null;
+      IList replacementValues = GetReplacementValues(modifiableDataset, variableName, model, rows, targetValues, out originalValues, replacementMethod, factorReplacementMethod);
+
+      double newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, replacementValues, targetValues);
+      double impact = quality - newValue;
+
+      return impact;
+    }
+
+    private static IList GetReplacementValues(ModifiableDataset modifiableDataset,
+      string variableName,
+      IClassificationModel model,
+      IEnumerable<int> rows,
+      IEnumerable<double> targetValues,
+      out IList originalValues,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
       FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
 
-      var problemData = solution.ProblemData;
-      var dataset = problemData.Dataset;
-      var model = (IClassificationModel)solution.Model.Clone(); //mkommend: clone of model is necessary, because the thresholds for IDiscriminantClassificationModels are updated
-
-      IEnumerable<int> rows;
-      IEnumerable<double> targetValues;
-      double originalAccuracy;
-
-      OnlineCalculatorError error;
-
-      switch (data) {
-        case DataPartitionEnum.All:
-          rows = problemData.AllIndices;
-          targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.AllIndices).ToList();
-          originalAccuracy = OnlineAccuracyCalculator.Calculate(targetValues, solution.EstimatedClassValues, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during accuracy calculation.");
-          break;
-        case DataPartitionEnum.Training:
-          rows = problemData.TrainingIndices;
-          targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToList();
-          originalAccuracy = OnlineAccuracyCalculator.Calculate(targetValues, solution.EstimatedTrainingClassValues, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during accuracy calculation.");
-          break;
-        case DataPartitionEnum.Test:
-          rows = problemData.TestIndices;
-          targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TestIndices).ToList();
-          originalAccuracy = OnlineAccuracyCalculator.Calculate(targetValues, solution.EstimatedTestClassValues, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during accuracy calculation.");
-          break;
-        default: throw new ArgumentException(string.Format("DataPartition {0} cannot be handled.", data));
-      }
-
-      var impacts = new Dictionary<string, double>();
-      var modifiableDataset = ((Dataset)dataset).ToModifiable();
-
-      var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(solution.Model.VariablesUsedForPrediction));
-      var allowedInputVariables = dataset.VariableNames.Where(v => inputvariables.Contains(v)).ToList();
-
-      // calculate impacts for double variables
-      foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<double>)) {
-        var newEstimates = EvaluateModelWithReplacedVariable(model, inputVariable, modifiableDataset, rows, replacementMethod);
-        var newAccuracy = OnlineAccuracyCalculator.Calculate(targetValues, newEstimates, out error);
-        if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
-
-        impacts[inputVariable] = originalAccuracy - newAccuracy;
-      }
-
-      // calculate impacts for string variables
-      foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<string>)) {
-        if (factorReplacementMethod == FactorReplacementMethodEnum.Best) {
-          // try replacing with all possible values and find the best replacement value
-          var smallestImpact = double.PositiveInfinity;
-          foreach (var repl in problemData.Dataset.GetStringValues(inputVariable, rows).Distinct()) {
-            var newEstimates = EvaluateModelWithReplacedVariable(model, inputVariable, modifiableDataset, rows,
-              Enumerable.Repeat(repl, dataset.Rows));
-            var newAccuracy = OnlineAccuracyCalculator.Calculate(targetValues, newEstimates, out error);
-            if (error != OnlineCalculatorError.None)
-              throw new InvalidOperationException("Error during accuracy calculation with replaced inputs.");
-
-            var impact = originalAccuracy - newAccuracy;
-            if (impact < smallestImpact) smallestImpact = impact;
-          }
-          impacts[inputVariable] = smallestImpact;
-        } else {
-          // for replacement methods shuffle and mode
-          // calculate impacts for factor variables
-
-          var newEstimates = EvaluateModelWithReplacedVariable(model, inputVariable, modifiableDataset, rows,
-            factorReplacementMethod);
-          var newAccuracy = OnlineAccuracyCalculator.Calculate(targetValues, newEstimates, out error);
-          if (error != OnlineCalculatorError.None)
-            throw new InvalidOperationException("Error during accuracy calculation with replaced inputs.");
-
-          impacts[inputVariable] = originalAccuracy - newAccuracy;
-        }
-      } // foreach
-      return impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value));
-    }
-
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IClassificationModel model, string variable, ModifiableDataset dataset, IEnumerable<int> rows, ReplacementMethodEnum replacement = ReplacementMethodEnum.Median) {
-      var originalValues = dataset.GetReadOnlyDoubleValues(variable).ToList();
+      IList replacementValues = null;
+      if (modifiableDataset.VariableHasType<double>(variableName)) {
+        originalValues = modifiableDataset.GetReadOnlyDoubleValues(variableName).ToList();
+        replacementValues = GetReplacementValuesForDouble(modifiableDataset, rows, (List<double>)originalValues, replacementMethod);
+      } else if (modifiableDataset.VariableHasType<string>(variableName)) {
+        originalValues = modifiableDataset.GetReadOnlyStringValues(variableName).ToList();
+        replacementValues = GetReplacementValuesForString(model, modifiableDataset, variableName, rows, (List<string>)originalValues, targetValues, factorReplacementMethod);
+      } else {
+        throw new NotSupportedException("Variable not supported");
+      }
+
+      return replacementValues;
+    }
+
+    private static IList GetReplacementValuesForDouble(ModifiableDataset modifiableDataset,
+      IEnumerable<int> rows,
+      List<double> originalValues,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle) {
+
+      IRandom random = new FastRandom(31415);
+      List<double> replacementValues;
       double replacementValue;
-      List<double> replacementValues;
-      IRandom rand;
-
-      switch (replacement) {
+
+      switch (replacementMethod) {
         case ReplacementMethodEnum.Median:
           replacementValue = rows.Select(r => originalValues[r]).Median();
-          replacementValues = Enumerable.Repeat(replacementValue, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
           break;
         case ReplacementMethodEnum.Average:
           replacementValue = rows.Select(r => originalValues[r]).Average();
-          replacementValues = Enumerable.Repeat(replacementValue, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
           break;
         case ReplacementMethodEnum.Shuffle:
           // new var has same empirical distribution but the relation to y is broken
-          rand = new FastRandom(31415);
           // prepare a complete column for the dataset
-          replacementValues = Enumerable.Repeat(double.NaN, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
           // shuffle only the selected rows
-          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(rand).ToList();
+          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
           int i = 0;
           // update column values 
@@ -208 +232 @@
           var avg = rows.Select(r => originalValues[r]).Average();
           var stdDev = rows.Select(r => originalValues[r]).StandardDeviation();
-          rand = new FastRandom(31415);
           // prepare a complete column for the dataset
-          replacementValues = Enumerable.Repeat(double.NaN, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
           // update column values 
           foreach (var r in rows) {
-            replacementValues[r] = NormalDistributedRandom.NextDouble(rand, avg, stdDev);
+            replacementValues[r] = NormalDistributedRandom.NextDouble(random, avg, stdDev);
           }
           break;
 
         default:
-          throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacement));
-      }
-
-      return EvaluateModelWithReplacedVariable(model, variable, dataset, rows, replacementValues);
-    }
-
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(
-      IClassificationModel model, string variable, ModifiableDataset dataset,
-      IEnumerable<int> rows,
-      FactorReplacementMethodEnum replacement = FactorReplacementMethodEnum.Shuffle) {
-      var originalValues = dataset.GetReadOnlyStringValues(variable).ToList();
-      List<string> replacementValues;
-      IRandom rand;
-
-      switch (replacement) {
+          throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacementMethod));
+      }
+
+      return replacementValues;
+    }
+
+    private static IList GetReplacementValuesForString(IClassificationModel model,
+      ModifiableDataset modifiableDataset,
+      string variableName,
+      IEnumerable<int> rows,
+      List<string> originalValues,
+      IEnumerable<double> targetValues,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Shuffle) {
+
+      List<string> replacementValues = null;
+      IRandom random = new FastRandom(31415);
+
+      switch (factorReplacementMethod) {
+        case FactorReplacementMethodEnum.Best:
+          // try replacing with all possible values and find the best replacement value
+          var bestQuality = double.NegativeInfinity;
+          foreach (var repl in modifiableDataset.GetStringValues(variableName, rows).Distinct()) {
+            List<string> curReplacementValues = Enumerable.Repeat(repl, modifiableDataset.Rows).ToList();
+            //fholzing: this result could be used later on (theoretically), but is neglected for better readability/method consistency 
+            var newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, curReplacementValues, targetValues);
+            var curQuality = newValue;
+
+            if (curQuality > bestQuality) {
+              bestQuality = curQuality;
+              replacementValues = curReplacementValues;
+            }
+          }
+          break;
         case FactorReplacementMethodEnum.Mode:
           var mostCommonValue = rows.Select(r => originalValues[r])
@@ -238 +279 @@
             .OrderByDescending(g => g.Count())
             .First().Key;
-          replacementValues = Enumerable.Repeat(mostCommonValue, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(mostCommonValue, modifiableDataset.Rows).ToList();
           break;
         case FactorReplacementMethodEnum.Shuffle:
           // new var has same empirical distribution but the relation to y is broken
-          rand = new FastRandom(31415);
           // prepare a complete column for the dataset
-          replacementValues = Enumerable.Repeat(string.Empty, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(string.Empty, modifiableDataset.Rows).ToList();
           // shuffle only the selected rows
-          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(rand).ToList();
+          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
           int i = 0;
           // update column values 
@@ -254 +294 @@
           break;
         default:
-          throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", replacement));
-      }
-
-      return EvaluateModelWithReplacedVariable(model, variable, dataset, rows, replacementValues);
-    }
-
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IClassificationModel model, string variable,
-      ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<double> replacementValues) {
-      var originalValues = dataset.GetReadOnlyDoubleValues(variable).ToList();
-      dataset.ReplaceVariable(variable, replacementValues.ToList());
-
+          throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", factorReplacementMethod));
+      }
+
+      return replacementValues;
+    }
+
+    private static double CalculateQualityForReplacement(
+      IClassificationModel model,
+      ModifiableDataset modifiableDataset,
+      string variableName,
+      IList originalValues,
+      IEnumerable<int> rows,
+      IList replacementValues,
+      IEnumerable<double> targetValues) {
+
+      modifiableDataset.ReplaceVariable(variableName, replacementValues);
       var discModel = model as IDiscriminantFunctionClassificationModel;
       if (discModel != null) {
-        var problemData = new ClassificationProblemData(dataset, dataset.VariableNames, model.TargetVariable);
+        var problemData = new ClassificationProblemData(modifiableDataset, modifiableDataset.VariableNames, model.TargetVariable);
         discModel.RecalculateModelParameters(problemData, rows);
       }
 
       //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements
-      var estimates = model.GetEstimatedClassValues(dataset, rows).ToList();
-      dataset.ReplaceVariable(variable, originalValues);
-
-      return estimates;
-    }
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IClassificationModel model, string variable,
-      ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<string> replacementValues) {
-      var originalValues = dataset.GetReadOnlyStringValues(variable).ToList();
-      dataset.ReplaceVariable(variable, replacementValues.ToList());
-
-
-      var discModel = model as IDiscriminantFunctionClassificationModel;
-      if (discModel != null) {
-        var problemData = new ClassificationProblemData(dataset, dataset.VariableNames, model.TargetVariable);
-        discModel.RecalculateModelParameters(problemData, rows);
-      }
-
-      //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements
-      var estimates = model.GetEstimatedClassValues(dataset, rows).ToList();
-      dataset.ReplaceVariable(variable, originalValues);
-
-      return estimates;
+      var estimates = model.GetEstimatedClassValues(modifiableDataset, rows).ToList();
+      var ret = CalculateQuality(targetValues, estimates);
+      modifiableDataset.ReplaceVariable(variableName, originalValues);
+
+      return ret;
+    }
+
+    public static double CalculateQuality(IEnumerable<double> targetValues, IEnumerable<double> estimatedClassValues) {
+      OnlineCalculatorError errorState;
+      var ret = OnlineAccuracyCalculator.Calculate(targetValues, estimatedClassValues, out errorState);
+      if (errorState != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during calculation with replaced inputs."); }
+      return ret;
+    }
+
+    public static IEnumerable<int> GetPartitionRows(DataPartitionEnum dataPartition, IClassificationProblemData problemData) {
+      IEnumerable<int> rows;
+
+      switch (dataPartition) {
+        case DataPartitionEnum.All:
+          rows = problemData.AllIndices;
+          break;
+        case DataPartitionEnum.Test:
+          rows = problemData.TestIndices;
+          break;
+        case DataPartitionEnum.Training:
+          rows = problemData.TrainingIndices;
+          break;
+        default:
+          throw new NotSupportedException("DataPartition not supported");
+      }
+
+      return rows;
     }
   }

trunk/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolutionVariableImpactsCalculator.cs

@@ -23 +23 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
@@ -36 +37 @@
   [Item("RegressionSolution Impacts Calculator", "Calculation of the impacts of input variables for any regression solution")]
   public sealed class RegressionSolutionVariableImpactsCalculator : ParameterizedNamedItem {
+    #region Parameters/Properties
     public enum ReplacementMethodEnum {
       Median,
@@ -54 +56 @@
 
     private const string ReplacementParameterName = "Replacement Method";
+    private const string FactorReplacementParameterName = "Factor Replacement Method";
     private const string DataPartitionParameterName = "DataPartition";
 
     public IFixedValueParameter<EnumValue<ReplacementMethodEnum>> ReplacementParameter {
       get { return (IFixedValueParameter<EnumValue<ReplacementMethodEnum>>)Parameters[ReplacementParameterName]; }
+    }
+    public IFixedValueParameter<EnumValue<FactorReplacementMethodEnum>> FactorReplacementParameter {
+      get { return (IFixedValueParameter<EnumValue<FactorReplacementMethodEnum>>)Parameters[FactorReplacementParameterName]; }
     }
     public IFixedValueParameter<EnumValue<DataPartitionEnum>> DataPartitionParameter {
@@ -67 +73 @@
       set { ReplacementParameter.Value.Value = value; }
     }
+    public FactorReplacementMethodEnum FactorReplacementMethod {
+      get { return FactorReplacementParameter.Value.Value; }
+      set { FactorReplacementParameter.Value.Value = value; }
+    }
     public DataPartitionEnum DataPartition {
       get { return DataPartitionParameter.Value.Value; }
       set { DataPartitionParameter.Value.Value = value; }
     }
-
-
+    #endregion
+
+    #region Ctor/Cloner
     [StorableConstructor]
     private RegressionSolutionVariableImpactsCalculator(bool deserializing) : base(deserializing) { }
     private RegressionSolutionVariableImpactsCalculator(RegressionSolutionVariableImpactsCalculator original, Cloner cloner)
       : base(original, cloner) { }
+    public RegressionSolutionVariableImpactsCalculator()
+      : base() {
+      Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum.Shuffle)));
+      Parameters.Add(new FixedValueParameter<EnumValue<FactorReplacementMethodEnum>>(FactorReplacementParameterName, "The replacement method for factor variables during impact calculation.", new EnumValue<FactorReplacementMethodEnum>(FactorReplacementMethodEnum.Best)));
+      Parameters.Add(new FixedValueParameter<EnumValue<DataPartitionEnum>>(DataPartitionParameterName, "The data partition on which the impacts are calculated.", new EnumValue<DataPartitionEnum>(DataPartitionEnum.Training)));
+    }
+
     public override IDeepCloneable Clone(Cloner cloner) {
       return new RegressionSolutionVariableImpactsCalculator(this, cloner);
     }
-
-    public RegressionSolutionVariableImpactsCalculator()
-      : base() {
-      Parameters.Add(new FixedValueParameter<EnumValue<ReplacementMethodEnum>>(ReplacementParameterName, "The replacement method for variables during impact calculation.", new EnumValue<ReplacementMethodEnum>(ReplacementMethodEnum.Median)));
-      Parameters.Add(new FixedValueParameter<EnumValue<DataPartitionEnum>>(DataPartitionParameterName, "The data partition on which the impacts are calculated.", new EnumValue<DataPartitionEnum>(DataPartitionEnum.Training)));
-    }
+    #endregion
 
     //mkommend: annoying name clash with static method, open to better naming suggestions
     public IEnumerable<Tuple<string, double>> Calculate(IRegressionSolution solution) {
-      return CalculateImpacts(solution, DataPartition, ReplacementMethod);
+      return CalculateImpacts(solution, ReplacementMethod, FactorReplacementMethod, DataPartition);
     }
 
     public static IEnumerable<Tuple<string, double>> CalculateImpacts(
       IRegressionSolution solution,
-      DataPartitionEnum data = DataPartitionEnum.Training,
-      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
       FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
-      Func<double, string, bool> progressCallback = null) {
-
-      var problemData = solution.ProblemData;
-      var dataset = problemData.Dataset;
-
-      IEnumerable<int> rows;
-      IEnumerable<double> targetValues;
-      double originalR2 = -1;
-
-      OnlineCalculatorError error;
-
-      switch (data) {
-        case DataPartitionEnum.All:
-          rows = solution.ProblemData.AllIndices;
-          targetValues = problemData.TargetVariableValues.ToList();
-          originalR2 = OnlinePearsonsRCalculator.Calculate(problemData.TargetVariableValues, solution.EstimatedValues, out error);
-          if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation.");
-          originalR2 = originalR2 * originalR2;
-          break;
-        case DataPartitionEnum.Training:
-          rows = problemData.TrainingIndices;
-          targetValues = problemData.TargetVariableTrainingValues.ToList();
-          originalR2 = solution.TrainingRSquared;
-          break;
-        case DataPartitionEnum.Test:
-          rows = problemData.TestIndices;
-          targetValues = problemData.TargetVariableTestValues.ToList();
-          originalR2 = solution.TestRSquared;
-          break;
-        default: throw new ArgumentException(string.Format("DataPartition {0} cannot be handled.", data));
-      }
+      DataPartitionEnum dataPartition = DataPartitionEnum.Training) {
+
+      IEnumerable<int> rows = GetPartitionRows(dataPartition, solution.ProblemData);
+      IEnumerable<double> estimatedValues = solution.GetEstimatedValues(rows);
+      return CalculateImpacts(solution.Model, solution.ProblemData, estimatedValues, rows, replacementMethod, factorReplacementMethod);
+    }
+
+    public static IEnumerable<Tuple<string, double>> CalculateImpacts(
+     IRegressionModel model,
+     IRegressionProblemData problemData,
+     IEnumerable<double> estimatedValues,
+     IEnumerable<int> rows,
+     ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
+     FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
+
+      //fholzing: try and catch in case a different dataset is loaded, otherwise statement is neglectable
+      var missingVariables = model.VariablesUsedForPrediction.Except(problemData.Dataset.VariableNames);
+      if (missingVariables.Any()) {
+        throw new InvalidOperationException(string.Format("Can not calculate variable impacts, because the model uses inputs missing in the dataset ({0})", string.Join(", ", missingVariables)));
+      }
+      IEnumerable<double> targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      var originalQuality = CalculateQuality(targetValues, estimatedValues);
 
       var impacts = new Dictionary<string, double>();
-      var modifiableDataset = ((Dataset)dataset).ToModifiable();
-
-      var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(solution.Model.VariablesUsedForPrediction));
-      var allowedInputVariables = dataset.VariableNames.Where(v => inputvariables.Contains(v)).ToList();
-
-      int curIdx = 0;
-      int count = allowedInputVariables.Where(problemData.Dataset.VariableHasType<double>).Count();
-      // calculate impacts for double variables
-      foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<double>)) {
-        //Report the current progress in percent. If the callback returns true, it means the execution shall be stopped
-        if (progressCallback != null) {
-          curIdx++;
-          if (progressCallback((double)curIdx / count, string.Format("Calculating impact for variable {0} ({1} of {2})", inputVariable, curIdx, count))) { return null; }
-        }
-        var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows, replacementMethod);
-        var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-        if (error != OnlineCalculatorError.None) throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
-
-        newR2 = newR2 * newR2;
-        var impact = originalR2 - newR2;
-        impacts[inputVariable] = impact;
-      }
-
-      // calculate impacts for string variables
-      foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType<string>)) {
-        if (factorReplacementMethod == FactorReplacementMethodEnum.Best) {
-          // try replacing with all possible values and find the best replacement value
-          var smallestImpact = double.PositiveInfinity;
-          foreach (var repl in problemData.Dataset.GetStringValues(inputVariable, rows).Distinct()) {
-            var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows,
-              Enumerable.Repeat(repl, dataset.Rows));
-            var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-            if (error != OnlineCalculatorError.None)
-              throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
-
-            newR2 = newR2 * newR2;
-            var impact = originalR2 - newR2;
-            if (impact < smallestImpact) smallestImpact = impact;
-          }
-          impacts[inputVariable] = smallestImpact;
-        } else {
-          // for replacement methods shuffle and mode
-          // calculate impacts for factor variables
-
-          var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows,
-            factorReplacementMethod);
-          var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
-          if (error != OnlineCalculatorError.None)
-            throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
-
-          newR2 = newR2 * newR2;
-          var impact = originalR2 - newR2;
-          impacts[inputVariable] = impact;
-        }
-      } // foreach
-      return impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value));
-    }
-
-
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable, ModifiableDataset dataset, IEnumerable<int> rows, ReplacementMethodEnum replacement = ReplacementMethodEnum.Median) {
-      var originalValues = dataset.GetReadOnlyDoubleValues(variable).ToList();
+      var inputvariables = new HashSet<string>(problemData.AllowedInputVariables.Union(model.VariablesUsedForPrediction));
+      var modifiableDataset = ((Dataset)(problemData.Dataset).Clone()).ToModifiable();
+
+      foreach (var inputVariable in inputvariables) {
+        impacts[inputVariable] = CalculateImpact(inputVariable, model, problemData, modifiableDataset, rows, replacementMethod, factorReplacementMethod, targetValues, originalQuality);
+      }
+
+      return impacts.Select(i => Tuple.Create(i.Key, i.Value));
+    }
+
+    public static double CalculateImpact(string variableName,
+      IRegressionModel model,
+      IRegressionProblemData problemData,
+      ModifiableDataset modifiableDataset,
+      IEnumerable<int> rows,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best,
+      IEnumerable<double> targetValues = null,
+      double quality = double.NaN) {
+
+      if (!model.VariablesUsedForPrediction.Contains(variableName)) { return 0.0; }
+      if (!problemData.Dataset.VariableNames.Contains(variableName)) {
+        throw new InvalidOperationException(string.Format("Can not calculate variable impact, because the model uses inputs missing in the dataset ({0})", variableName));
+      }
+
+      if (targetValues == null) {
+        targetValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows);
+      }
+      if (quality == double.NaN) {
+        quality = CalculateQuality(model.GetEstimatedValues(modifiableDataset, rows), targetValues);
+      }
+
+      IList originalValues = null;
+      IList replacementValues = GetReplacementValues(modifiableDataset, variableName, model, rows, targetValues, out originalValues, replacementMethod, factorReplacementMethod);
+
+      double newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, replacementValues, targetValues);
+      double impact = quality - newValue;
+
+      return impact;
+    }
+
+    private static IList GetReplacementValues(ModifiableDataset modifiableDataset,
+      string variableName,
+      IRegressionModel model,
+      IEnumerable<int> rows,
+      IEnumerable<double> targetValues,
+      out IList originalValues,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best) {
+
+      IList replacementValues = null;
+      if (modifiableDataset.VariableHasType<double>(variableName)) {
+        originalValues = modifiableDataset.GetReadOnlyDoubleValues(variableName).ToList();
+        replacementValues = GetReplacementValuesForDouble(modifiableDataset, rows, (List<double>)originalValues, replacementMethod);
+      } else if (modifiableDataset.VariableHasType<string>(variableName)) {
+        originalValues = modifiableDataset.GetReadOnlyStringValues(variableName).ToList();
+        replacementValues = GetReplacementValuesForString(model, modifiableDataset, variableName, rows, (List<string>)originalValues, targetValues, factorReplacementMethod);
+      } else {
+        throw new NotSupportedException("Variable not supported");
+      }
+
+      return replacementValues;
+    }
+
+    private static IList GetReplacementValuesForDouble(ModifiableDataset modifiableDataset,
+      IEnumerable<int> rows,
+      List<double> originalValues,
+      ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Shuffle) {
+
+      IRandom random = new FastRandom(31415);
+      List<double> replacementValues;
       double replacementValue;
-      List<double> replacementValues;
-      IRandom rand;
-
-      switch (replacement) {
+
+      switch (replacementMethod) {
         case ReplacementMethodEnum.Median:
           replacementValue = rows.Select(r => originalValues[r]).Median();
-          replacementValues = Enumerable.Repeat(replacementValue, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
           break;
         case ReplacementMethodEnum.Average:
           replacementValue = rows.Select(r => originalValues[r]).Average();
-          replacementValues = Enumerable.Repeat(replacementValue, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(replacementValue, modifiableDataset.Rows).ToList();
           break;
         case ReplacementMethodEnum.Shuffle:
           // new var has same empirical distribution but the relation to y is broken
-          rand = new FastRandom(31415);
           // prepare a complete column for the dataset
-          replacementValues = Enumerable.Repeat(double.NaN, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
           // shuffle only the selected rows
-          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(rand).ToList();
+          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
           int i = 0;
           // update column values 
@@ -220 +230 @@
           var avg = rows.Select(r => originalValues[r]).Average();
           var stdDev = rows.Select(r => originalValues[r]).StandardDeviation();
-          rand = new FastRandom(31415);
           // prepare a complete column for the dataset
-          replacementValues = Enumerable.Repeat(double.NaN, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(double.NaN, modifiableDataset.Rows).ToList();
           // update column values 
           foreach (var r in rows) {
-            replacementValues[r] = NormalDistributedRandom.NextDouble(rand, avg, stdDev);
+            replacementValues[r] = NormalDistributedRandom.NextDouble(random, avg, stdDev);
           }
           break;
 
         default:
-          throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacement));
-      }
-
-      return EvaluateModelWithReplacedVariable(model, variable, dataset, rows, replacementValues);
-    }
-
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(
-      IRegressionModel model, string variable, ModifiableDataset dataset,
-      IEnumerable<int> rows,
-      FactorReplacementMethodEnum replacement = FactorReplacementMethodEnum.Shuffle) {
-      var originalValues = dataset.GetReadOnlyStringValues(variable).ToList();
-      List<string> replacementValues;
-      IRandom rand;
-
-      switch (replacement) {
+          throw new ArgumentException(string.Format("ReplacementMethod {0} cannot be handled.", replacementMethod));
+      }
+
+      return replacementValues;
+    }
+
+    private static IList GetReplacementValuesForString(IRegressionModel model,
+      ModifiableDataset modifiableDataset,
+      string variableName,
+      IEnumerable<int> rows,
+      List<string> originalValues,
+      IEnumerable<double> targetValues,
+      FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Shuffle) {
+
+      List<string> replacementValues = null;
+      IRandom random = new FastRandom(31415);
+
+      switch (factorReplacementMethod) {
+        case FactorReplacementMethodEnum.Best:
+          // try replacing with all possible values and find the best replacement value
+          var bestQuality = double.NegativeInfinity;
+          foreach (var repl in modifiableDataset.GetStringValues(variableName, rows).Distinct()) {
+            List<string> curReplacementValues = Enumerable.Repeat(repl, modifiableDataset.Rows).ToList();
+            //fholzing: this result could be used later on (theoretically), but is neglected for better readability/method consistency 
+            var newValue = CalculateQualityForReplacement(model, modifiableDataset, variableName, originalValues, rows, curReplacementValues, targetValues);
+            var curQuality = newValue;
+
+            if (curQuality > bestQuality) {
+              bestQuality = curQuality;
+              replacementValues = curReplacementValues;
+            }
+          }
+          break;
         case FactorReplacementMethodEnum.Mode:
           var mostCommonValue = rows.Select(r => originalValues[r])
@@ -250 +277 @@
             .OrderByDescending(g => g.Count())
             .First().Key;
-          replacementValues = Enumerable.Repeat(mostCommonValue, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(mostCommonValue, modifiableDataset.Rows).ToList();
           break;
         case FactorReplacementMethodEnum.Shuffle:
           // new var has same empirical distribution but the relation to y is broken
-          rand = new FastRandom(31415);
           // prepare a complete column for the dataset
-          replacementValues = Enumerable.Repeat(string.Empty, dataset.Rows).ToList();
+          replacementValues = Enumerable.Repeat(string.Empty, modifiableDataset.Rows).ToList();
           // shuffle only the selected rows
-          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(rand).ToList();
+          var shuffledValues = rows.Select(r => originalValues[r]).Shuffle(random).ToList();
           int i = 0;
           // update column values 
@@ -266 +292 @@
           break;
         default:
-          throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", replacement));
-      }
-
-      return EvaluateModelWithReplacedVariable(model, variable, dataset, rows, replacementValues);
-    }
-
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable,
-      ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<double> replacementValues) {
-      var originalValues = dataset.GetReadOnlyDoubleValues(variable).ToList();
-      dataset.ReplaceVariable(variable, replacementValues.ToList());
+          throw new ArgumentException(string.Format("FactorReplacementMethod {0} cannot be handled.", factorReplacementMethod));
+      }
+
+      return replacementValues;
+    }
+
+    private static double CalculateQualityForReplacement(
+      IRegressionModel model,
+      ModifiableDataset modifiableDataset,
+      string variableName,
+      IList originalValues,
+      IEnumerable<int> rows,
+      IList replacementValues,
+      IEnumerable<double> targetValues) {
+
+      modifiableDataset.ReplaceVariable(variableName, replacementValues);
       //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements
-      var estimates = model.GetEstimatedValues(dataset, rows).ToList();
-      dataset.ReplaceVariable(variable, originalValues);
-
-      return estimates;
-    }
-    private static IEnumerable<double> EvaluateModelWithReplacedVariable(IRegressionModel model, string variable,
-      ModifiableDataset dataset, IEnumerable<int> rows, IEnumerable<string> replacementValues) {
-      var originalValues = dataset.GetReadOnlyStringValues(variable).ToList();
-      dataset.ReplaceVariable(variable, replacementValues.ToList());
-      //mkommend: ToList is used on purpose to avoid lazy evaluation that could result in wrong estimates due to variable replacements
-      var estimates = model.GetEstimatedValues(dataset, rows).ToList();
-      dataset.ReplaceVariable(variable, originalValues);
-
-      return estimates;
+      var estimates = model.GetEstimatedValues(modifiableDataset, rows).ToList();
+      var ret = CalculateQuality(targetValues, estimates);
+      modifiableDataset.ReplaceVariable(variableName, originalValues);
+
+      return ret;
+    }
+
+    public static double CalculateQuality(IEnumerable<double> targetValues, IEnumerable<double> estimatedValues) {
+      OnlineCalculatorError errorState;
+      var ret = OnlinePearsonsRCalculator.Calculate(targetValues, estimatedValues, out errorState);
+      if (errorState != OnlineCalculatorError.None) { throw new InvalidOperationException("Error during calculation with replaced inputs."); }
+      return ret * ret;
+    }
+
+    public static IEnumerable<int> GetPartitionRows(DataPartitionEnum dataPartition, IRegressionProblemData problemData) {
+      IEnumerable<int> rows;
+
+      switch (dataPartition) {
+        case DataPartitionEnum.All:
+          rows = problemData.AllIndices;
+          break;
+        case DataPartitionEnum.Test:
+          rows = problemData.TestIndices;
+          break;
+        case DataPartitionEnum.Training:
+          rows = problemData.TrainingIndices;
+          break;
+        default:
+          throw new NotSupportedException("DataPartition not supported");
+      }
+
+      return rows;
     }
   }

trunk/HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4/IntervalInterpreterTest.cs

@@ -1 +1 @@
 using System;
 using System.Collections.Generic;
-using HeuristicLab.Problems.DataAnalysis;
-using HeuristicLab.Problems.DataAnalysis.Symbolic;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 

trunk/HeuristicLab.Tests/HeuristicLab.Tests.csproj

@@ -583 +583 @@
     <Compile Include="HeuristicLab.Persistence-3.3\UseCases.cs" />
     <Compile Include="HeuristicLab.PluginInfraStructure-3.3\TypeExtensionsTest.cs" />
+    <Compile Include="HeuristicLab.Problems.DataAnalysis-3.4\ClassificationVariableImpactCalculationTest.cs" />
     <Compile Include="HeuristicLab.Problems.DataAnalysis-3.4\IntervalTest.cs" />
+    <Compile Include="HeuristicLab.Problems.DataAnalysis-3.4\RegressionVariableImpactCalculationTest.cs" />
     <Compile Include="HeuristicLab.Problems.DataAnalysis-3.4\ThresholdCalculatorsTest.cs" />
     <Compile Include="HeuristicLab.Problems.DataAnalysis-3.4\OnlineCalculatorPerformanceTest.cs" />
     <Compile Include="HeuristicLab.Problems.DataAnalysis-3.4\StatisticCalculatorsTest.cs" />
-    <Compile Include="HeuristicLab.Problems.DataAnalysis-3.4\VariableImpactCalculationTest.cs" />
     <Compile Include="HeuristicLab.Problems.DataAnalysis.Symbolic-3.4\DeriveTest.cs" />
     <Compile Include="HeuristicLab.Problems.DataAnalysis.Symbolic-3.4\InfixExpressionParserTest.cs" />