Changeset 8276 for branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis

Timestamp:

07/10/12 15:26:13 (12 years ago)

Author:

sforsten

Message:

#1292:

• merged r8034:8179 from trunk
• added BackgroundWorker
• added ProgressBar
• added SpearmansRankCorrelationCoefficientCalculator
• corrected bug in HoeffdingsDependenceCalculator
• made some changes in the GUI

Location:

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis

Files:

• . (modified) (1 prop)
• 3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj (modified) (2 diffs)
• 3.4/Implementation/Classification/ClassificationEnsembleSolution.cs (modified) (10 diffs)
• 3.4/Implementation/Classification/ClassificationProblemData.cs (modified) (1 diff)
• 3.4/Implementation/Classification/ClassificationSolution.cs (modified) (2 diffs)
• 3.4/Implementation/Classification/ClassificationSolutionBase.cs (modified) (1 diff)
• 3.4/Implementation/Classification/DiscriminantFunctionClassificationSolution.cs (modified) (2 diffs)
• 3.4/Implementation/Classification/DiscriminantFunctionClassificationSolutionBase.cs (modified) (3 diffs)
• 3.4/Implementation/Classification/ThresholdCalculators/AccuracyMaximizationThresholdCalculator.cs (modified) (2 diffs)
• 3.4/Implementation/Clustering/ClusteringSolution.cs (modified) (2 diffs)
• 3.4/Implementation/DataAnalysisProblemData.cs (modified) (5 diffs)
• 3.4/Implementation/ExtendedHeatMap.cs (added)
• 3.4/Implementation/Regression/RegressionEnsembleSolution.cs (modified) (11 diffs)
• 3.4/Implementation/Regression/RegressionProblemData.cs (modified) (1 diff)
• 3.4/Implementation/Regression/RegressionSolution.cs (modified) (1 diff)
• 3.4/Implementation/Regression/RegressionSolutionBase.cs (modified) (5 diffs)
• 3.4/Interfaces/IDataAnalysisProblemData.cs (modified) (1 diff)
• 3.4/Interfaces/IOnlineCalculator.cs (modified) (1 diff)
• 3.4/OnlineCalculators/HoeffdingsDependenceCalculator.cs (modified) (1 diff)
• 3.4/OnlineCalculators/OnlineLinearScalingParameterCalculator.cs (modified) (3 diffs)
• 3.4/OnlineCalculators/SpearmansRankCorrelationCoefficientCalculator.cs (added)

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj

@@ -155 +155 @@
     <Compile Include="Implementation\Clustering\ClusteringProblemData.cs" />
     <Compile Include="Implementation\Clustering\ClusteringSolution.cs" />
+    <Compile Include="Implementation\ExtendedHeatMap.cs" />
     <Compile Include="Implementation\Regression\ConstantRegressionModel.cs" />
     <Compile Include="Implementation\Regression\ConstantRegressionSolution.cs" />
@@ -213 +214 @@
     <Compile Include="OnlineCalculators\OnlinePearsonsRSquaredCalculator.cs" />
     <Compile Include="Implementation\Regression\RegressionSolution.cs" />
+    <Compile Include="OnlineCalculators\SpearmansRankCorrelationCoefficientCalculator.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Implementation\Classification\ThresholdCalculators\AccuracyMaximizationThresholdCalculator.cs" />

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationEnsembleSolution.cs

@@ -37 +37 @@
   [Creatable("Data Analysis - Ensembles")]
   public sealed class ClassificationEnsembleSolution : ClassificationSolution, IClassificationEnsembleSolution {
+    private readonly Dictionary<int, double> trainingEvaluationCache = new Dictionary<int, double>();
+    private readonly Dictionary<int, double> testEvaluationCache = new Dictionary<int, double>();
+
     public new IClassificationEnsembleModel Model {
       get { return (IClassificationEnsembleModel)base.Model; }
@@ -85 +88 @@
       }
 
+      trainingEvaluationCache = new Dictionary<int, double>(original.ProblemData.TrainingIndices.Count());
+      testEvaluationCache = new Dictionary<int, double>(original.ProblemData.TestIndices.Count());
+
       classificationSolutions = cloner.Clone(original.classificationSolutions);
       RegisterClassificationSolutionsEventHandler();
@@ -128 +134 @@
       }
 
+      trainingEvaluationCache = new Dictionary<int, double>(problemData.TrainingIndices.Count());
+      testEvaluationCache = new Dictionary<int, double>(problemData.TestIndices.Count());
+
       RegisterClassificationSolutionsEventHandler();
       classificationSolutions.AddRange(solutions);
@@ -148 +157 @@
     public override IEnumerable<double> EstimatedTrainingClassValues {
       get {
-        var rows = ProblemData.TrainingIndizes;
-        var estimatedValuesEnumerators = (from model in Model.Models
-                                          select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedClassValues(ProblemData.Dataset, rows).GetEnumerator() })
-                                         .ToList();
-        var rowsEnumerator = rows.GetEnumerator();
-        // aggregate to make sure that MoveNext is called for all enumerators 
-        while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
-          int currentRow = rowsEnumerator.Current;
-
-          var selectedEnumerators = from pair in estimatedValuesEnumerators
-                                    where RowIsTrainingForModel(currentRow, pair.Model) && !RowIsTestForModel(currentRow, pair.Model)
-                                    select pair.EstimatedValuesEnumerator;
-          yield return AggregateEstimatedClassValues(selectedEnumerators.Select(x => x.Current));
+        var rows = ProblemData.TrainingIndices;
+        var rowsToEvaluate = rows.Except(trainingEvaluationCache.Keys);
+        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, (r, m) => RowIsTrainingForModel(r, m) && !RowIsTestForModel(r, m)).GetEnumerator();
+
+        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+          trainingEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
         }
+
+        return rows.Select(row => trainingEvaluationCache[row]);
       }
     }
@@ -167 +172 @@
     public override IEnumerable<double> EstimatedTestClassValues {
       get {
-        var rows = ProblemData.TestIndizes;
-        var estimatedValuesEnumerators = (from model in Model.Models
-                                          select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedClassValues(ProblemData.Dataset, rows).GetEnumerator() })
-                                         .ToList();
-        var rowsEnumerator = ProblemData.TestIndizes.GetEnumerator();
-        // aggregate to make sure that MoveNext is called for all enumerators 
-        while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
-          int currentRow = rowsEnumerator.Current;
-
-          var selectedEnumerators = from pair in estimatedValuesEnumerators
-                                    where RowIsTestForModel(currentRow, pair.Model)
-                                    select pair.EstimatedValuesEnumerator;
-
-          yield return AggregateEstimatedClassValues(selectedEnumerators.Select(x => x.Current));
+        var rows = ProblemData.TestIndices;
+        var rowsToEvaluate = rows.Except(testEvaluationCache.Keys);
+        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, RowIsTestForModel).GetEnumerator();
+
+        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+          testEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
         }
+
+        return rows.Select(row => testEvaluationCache[row]);
+      }
+    }
+
+    private IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows, Func<int, IClassificationModel, bool> modelSelectionPredicate) {
+      var estimatedValuesEnumerators = (from model in Model.Models
+                                        select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedClassValues(ProblemData.Dataset, rows).GetEnumerator() })
+                                       .ToList();
+      var rowsEnumerator = rows.GetEnumerator();
+      // aggregate to make sure that MoveNext is called for all enumerators 
+      while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
+        int currentRow = rowsEnumerator.Current;
+
+        var selectedEnumerators = from pair in estimatedValuesEnumerators
+                                  where modelSelectionPredicate(currentRow, pair.Model)
+                                  select pair.EstimatedValuesEnumerator;
+
+        yield return AggregateEstimatedClassValues(selectedEnumerators.Select(x => x.Current));
       }
     }
@@ -196 +213 @@
 
     public override IEnumerable<double> GetEstimatedClassValues(IEnumerable<int> rows) {
-      return from xs in GetEstimatedClassValueVectors(ProblemData.Dataset, rows)
-             select AggregateEstimatedClassValues(xs);
+      var rowsToEvaluate = rows.Except(evaluationCache.Keys);
+      var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+      var valuesEnumerator = (from xs in GetEstimatedClassValueVectors(ProblemData.Dataset, rowsToEvaluate)
+                              select AggregateEstimatedClassValues(xs))
+                             .GetEnumerator();
+
+      while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+        evaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
+      }
+
+      return rows.Select(row => evaluationCache[row]);
     }
 
@@ -223 +249 @@
 
     protected override void OnProblemDataChanged() {
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
+
       IClassificationProblemData problemData = new ClassificationProblemData(ProblemData.Dataset,
                                                                      ProblemData.AllowedInputVariables,
@@ -251 +281 @@
     public void AddClassificationSolutions(IEnumerable<IClassificationSolution> solutions) {
       classificationSolutions.AddRange(solutions);
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
     public void RemoveClassificationSolutions(IEnumerable<IClassificationSolution> solutions) {
       classificationSolutions.RemoveRange(solutions);
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
 
@@ -275 +313 @@
       trainingPartitions[solution.Model] = solution.ProblemData.TrainingPartition;
       testPartitions[solution.Model] = solution.ProblemData.TestPartition;
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
 
@@ -282 +324 @@
       trainingPartitions.Remove(solution.Model);
       testPartitions.Remove(solution.Model);
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
   }

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationProblemData.cs

@@ -207 +207 @@
 
     #region parameter properties
-    public ConstrainedValueParameter<StringValue> TargetVariableParameter {
-      get { return (ConstrainedValueParameter<StringValue>)Parameters[TargetVariableParameterName]; }
+    public IConstrainedValueParameter<StringValue> TargetVariableParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[TargetVariableParameterName]; }
     }
     public IFixedValueParameter<StringMatrix> ClassNamesParameter {

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolution.cs

@@ -44 +44 @@
     public ClassificationSolution(IClassificationModel model, IClassificationProblemData problemData)
       : base(model, problemData) {
-      evaluationCache = new Dictionary<int, double>();
+      evaluationCache = new Dictionary<int, double>(problemData.Dataset.Rows);
     }
 
@@ -51 +51 @@
     }
     public override IEnumerable<double> EstimatedTrainingClassValues {
-      get { return GetEstimatedClassValues(ProblemData.TrainingIndizes); }
+      get { return GetEstimatedClassValues(ProblemData.TrainingIndices); }
     }
     public override IEnumerable<double> EstimatedTestClassValues {
-      get { return GetEstimatedClassValues(ProblemData.TestIndizes); }
+      get { return GetEstimatedClassValues(ProblemData.TestIndices); }
     }
 

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ClassificationSolutionBase.cs

@@ -87 +87 @@
     protected void CalculateResults() {
       double[] estimatedTrainingClassValues = EstimatedTrainingClassValues.ToArray(); // cache values
-      double[] originalTrainingClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndizes).ToArray();
+      double[] originalTrainingClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices).ToArray();
       double[] estimatedTestClassValues = EstimatedTestClassValues.ToArray(); // cache values
-      double[] originalTestClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndizes).ToArray();
+      double[] originalTestClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices).ToArray();
 
       OnlineCalculatorError errorState;

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolution.cs

@@ -59 +59 @@
     }
     public override IEnumerable<double> EstimatedTrainingClassValues {
-      get { return GetEstimatedClassValues(ProblemData.TrainingIndizes); }
+      get { return GetEstimatedClassValues(ProblemData.TrainingIndices); }
     }
     public override IEnumerable<double> EstimatedTestClassValues {
-      get { return GetEstimatedClassValues(ProblemData.TestIndizes); }
+      get { return GetEstimatedClassValues(ProblemData.TestIndices); }
     }
 
@@ -82 +82 @@
     }
     public override IEnumerable<double> EstimatedTrainingValues {
-      get { return GetEstimatedValues(ProblemData.TrainingIndizes); }
+      get { return GetEstimatedValues(ProblemData.TrainingIndices); }
     }
     public override IEnumerable<double> EstimatedTestValues {
-      get { return GetEstimatedValues(ProblemData.TestIndizes); }
+      get { return GetEstimatedValues(ProblemData.TestIndices); }
     }
 

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/DiscriminantFunctionClassificationSolutionBase.cs

@@ -103 +103 @@
     protected void CalculateRegressionResults() {
       double[] estimatedTrainingValues = EstimatedTrainingValues.ToArray(); // cache values
-      double[] originalTrainingValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndizes).ToArray();
+      double[] originalTrainingValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices).ToArray();
       double[] estimatedTestValues = EstimatedTestValues.ToArray(); // cache values
-      double[] originalTestValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndizes).ToArray();
+      double[] originalTestValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices).ToArray();
 
       OnlineCalculatorError errorState;
@@ -140 +140 @@
       double[] classValues;
       double[] thresholds;
-      var targetClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndizes);
+      var targetClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices);
       AccuracyMaximizationThresholdCalculator.CalculateThresholds(ProblemData, EstimatedTrainingValues, targetClassValues, out classValues, out thresholds);
 
@@ -149 +149 @@
       double[] classValues;
       double[] thresholds;
-      var targetClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndizes);
+      var targetClassValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices);
       NormalDistributionCutPointsThresholdCalculator.CalculateThresholds(ProblemData, EstimatedTrainingValues, targetClassValues, out classValues, out thresholds);
 

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Classification/ThresholdCalculators/AccuracyMaximizationThresholdCalculator.cs

@@ -54 +54 @@
     public static void CalculateThresholds(IClassificationProblemData problemData, IEnumerable<double> estimatedValues, IEnumerable<double> targetClassValues, out double[] classValues, out double[] thresholds) {
       int slices = 100;
+      double minThresholdInc = 10e-5; // necessary to prevent infinite loop when maxEstimated - minEstimated is effectively zero (constant model)
       List<double> estimatedValuesList = estimatedValues.ToList();
       double maxEstimatedValue = estimatedValuesList.Max();
       double minEstimatedValue = estimatedValuesList.Min();
-      double thresholdIncrement = (maxEstimatedValue - minEstimatedValue) / slices;
+      double thresholdIncrement = Math.Max((maxEstimatedValue - minEstimatedValue) / slices, minThresholdInc);
       var estimatedAndTargetValuePairs =
         estimatedValuesList.Zip(targetClassValues, (x, y) => new { EstimatedValue = x, TargetClassValue = y })
@@ -70 +71 @@
 
       // incrementally calculate accuracy of all possible thresholds
-      int[,] confusionMatrix = new int[nClasses, nClasses];
-
       for (int i = 1; i < thresholds.Length; i++) {
         double lowerThreshold = thresholds[i - 1];

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Clustering/ClusteringSolution.cs

@@ -68 +68 @@
     public virtual IEnumerable<int> TrainingClusterValues {
       get {
-        return GetClusterValues(ProblemData.TrainingIndizes);
+        return GetClusterValues(ProblemData.TrainingIndices);
       }
     }
@@ -74 +74 @@
     public virtual IEnumerable<int> TestClusterValues {
       get {
-        return GetClusterValues(ProblemData.TestIndizes);
+        return GetClusterValues(ProblemData.TestIndices);
       }
     }

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/DataAnalysisProblemData.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
-using HeuristicLab.Analysis;
 using HeuristicLab.Collections;
 using HeuristicLab.Common;
@@ -53 +52 @@
       get { return (IFixedValueParameter<IntRange>)Parameters[TestPartitionParameterName]; }
     }
-    public IFixedValueParameter<HeatMap> DatasetHeatMapParameter {
-      get { return (IFixedValueParameter<HeatMap>)Parameters[DatasetHeatMapParameterName]; }
+    public IFixedValueParameter<ExtendedHeatMap> DatasetHeatMapParameter {
+      get { return (IFixedValueParameter<ExtendedHeatMap>)Parameters[DatasetHeatMapParameterName]; }
     }
     #endregion
@@ -79 +78 @@
       get { return TestPartitionParameter.Value; }
     }
-    public HeatMap DatasetHeatMap {
+    public ExtendedHeatMap DatasetHeatMap {
       get { return DatasetHeatMapParameter.Value; }
     }
 
-    public virtual IEnumerable<int> TrainingIndizes {
+    public virtual IEnumerable<int> TrainingIndices {
       get {
         return Enumerable.Range(TrainingPartition.Start, Math.Max(0, TrainingPartition.End - TrainingPartition.Start))
@@ -89 +88 @@
       }
     }
-    public virtual IEnumerable<int> TestIndizes {
+    public virtual IEnumerable<int> TestIndices {
       get {
         return Enumerable.Range(TestPartition.Start, Math.Max(0, TestPartition.End - TestPartition.Start))
@@ -140 +139 @@
       Parameters.Add(new FixedValueParameter<IntRange>(TrainingPartitionParameterName, "", new IntRange(trainingPartitionStart, trainingPartitionEnd)));
       Parameters.Add(new FixedValueParameter<IntRange>(TestPartitionParameterName, "", new IntRange(testPartitionStart, testPartitionEnd)));
-      Parameters.Add(new FixedValueParameter<HeatMap>(DatasetHeatMapParameterName, "", CalculateHeatMap(dataset)));
+      Parameters.Add(new FixedValueParameter<ExtendedHeatMap>(DatasetHeatMapParameterName, "", new ExtendedHeatMap(this)));
 
       ((ValueParameter<Dataset>)DatasetParameter).ReactOnValueToStringChangedAndValueItemImageChanged = false;
       RegisterEventHandlers();
-    }
-
-    private HeatMap CalculateHeatMap(Dataset dataset) {
-      IList<string> doubleVariableNames = dataset.DoubleVariables.ToList();
-      OnlineCalculatorError error;
-      int length = doubleVariableNames.Count;
-      double[,] elements = new double[length, length];
-
-      for (int i = 0; i < length; i++) {
-        for (int j = 0; j < i + 1; j++) {
-          elements[i, j] = OnlinePearsonsRSquaredCalculator.Calculate(dataset.GetDoubleValues(doubleVariableNames[length - 1 - i]), dataset.GetDoubleValues(doubleVariableNames[j]), out error);
-          elements[j, i] = elements[i, j];
-          if (!error.Equals(OnlineCalculatorError.None)) {
-            throw new ArgumentException("Calculator returned " + error);
-          }
-        }
-      }
-      return new HeatMap(elements, "Hoeffdings Dependence");
     }
 

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionEnsembleSolution.cs

@@ -37 +37 @@
   [Creatable("Data Analysis - Ensembles")]
   public sealed class RegressionEnsembleSolution : RegressionSolution, IRegressionEnsembleSolution {
+    private readonly Dictionary<int, double> trainingEvaluationCache = new Dictionary<int, double>();
+    private readonly Dictionary<int, double> testEvaluationCache = new Dictionary<int, double>();
+
     public new IRegressionEnsembleModel Model {
       get { return (IRegressionEnsembleModel)base.Model; }
@@ -52 +55 @@
 
     [Storable]
-    private Dictionary<IRegressionModel, IntRange> trainingPartitions;
+    private readonly Dictionary<IRegressionModel, IntRange> trainingPartitions;
     [Storable]
-    private Dictionary<IRegressionModel, IntRange> testPartitions;
+    private readonly Dictionary<IRegressionModel, IntRange> testPartitions;
 
     [StorableConstructor]
@@ -86 +89 @@
       }
 
+      trainingEvaluationCache = new Dictionary<int, double>(original.ProblemData.TrainingIndices.Count());
+      testEvaluationCache = new Dictionary<int, double>(original.ProblemData.TestIndices.Count());
+
       regressionSolutions = cloner.Clone(original.regressionSolutions);
       RegisterRegressionSolutionsEventHandler();
@@ -133 +139 @@
       }
 
+      trainingEvaluationCache = new Dictionary<int, double>(problemData.TrainingIndices.Count());
+      testEvaluationCache = new Dictionary<int, double>(problemData.TestIndices.Count());
+
       RegisterRegressionSolutionsEventHandler();
       regressionSolutions.AddRange(solutions);
@@ -153 +162 @@
     public override IEnumerable<double> EstimatedTrainingValues {
       get {
-        var rows = ProblemData.TrainingIndizes;
-        var estimatedValuesEnumerators = (from model in Model.Models
-                                          select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedValues(ProblemData.Dataset, rows).GetEnumerator() })
-                                         .ToList();
-        var rowsEnumerator = rows.GetEnumerator();
-        // aggregate to make sure that MoveNext is called for all enumerators 
-        while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
-          int currentRow = rowsEnumerator.Current;
-
-          var selectedEnumerators = from pair in estimatedValuesEnumerators
-                                    where RowIsTrainingForModel(currentRow, pair.Model) && !RowIsTestForModel(currentRow, pair.Model)
-                                    select pair.EstimatedValuesEnumerator;
-          yield return AggregateEstimatedValues(selectedEnumerators.Select(x => x.Current));
+        var rows = ProblemData.TrainingIndices;
+        var rowsToEvaluate = rows.Except(trainingEvaluationCache.Keys);
+        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, (r, m) => RowIsTrainingForModel(r, m) && !RowIsTestForModel(r, m)).GetEnumerator();
+
+        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+          trainingEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
         }
+
+        return rows.Select(row => trainingEvaluationCache[row]);
       }
     }
@@ -172 +177 @@
     public override IEnumerable<double> EstimatedTestValues {
       get {
-        var rows = ProblemData.TestIndizes;
-        var estimatedValuesEnumerators = (from model in Model.Models
-                                          select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedValues(ProblemData.Dataset, rows).GetEnumerator() })
-                                         .ToList();
-        var rowsEnumerator = ProblemData.TestIndizes.GetEnumerator();
-        // aggregate to make sure that MoveNext is called for all enumerators 
-        while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
-          int currentRow = rowsEnumerator.Current;
-
-          var selectedEnumerators = from pair in estimatedValuesEnumerators
-                                    where RowIsTestForModel(currentRow, pair.Model)
-                                    select pair.EstimatedValuesEnumerator;
-
-          yield return AggregateEstimatedValues(selectedEnumerators.Select(x => x.Current));
+        var rows = ProblemData.TestIndices;
+        var rowsToEvaluate = rows.Except(testEvaluationCache.Keys);
+        var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+        var valuesEnumerator = GetEstimatedValues(rowsToEvaluate, RowIsTestForModel).GetEnumerator();
+
+        while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+          testEvaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
         }
+
+        return rows.Select(row => testEvaluationCache[row]);
+      }
+    }
+
+    private IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows, Func<int, IRegressionModel, bool> modelSelectionPredicate) {
+      var estimatedValuesEnumerators = (from model in Model.Models
+                                        select new { Model = model, EstimatedValuesEnumerator = model.GetEstimatedValues(ProblemData.Dataset, rows).GetEnumerator() })
+                                       .ToList();
+      var rowsEnumerator = rows.GetEnumerator();
+      // aggregate to make sure that MoveNext is called for all enumerators 
+      while (rowsEnumerator.MoveNext() & estimatedValuesEnumerators.Select(en => en.EstimatedValuesEnumerator.MoveNext()).Aggregate(true, (acc, b) => acc & b)) {
+        int currentRow = rowsEnumerator.Current;
+
+        var selectedEnumerators = from pair in estimatedValuesEnumerators
+                                  where modelSelectionPredicate(currentRow, pair.Model)
+                                  select pair.EstimatedValuesEnumerator;
+
+        yield return AggregateEstimatedValues(selectedEnumerators.Select(x => x.Current));
       }
     }
@@ -201 +218 @@
 
     public override IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows) {
-      return from xs in GetEstimatedValueVectors(ProblemData.Dataset, rows)
-             select AggregateEstimatedValues(xs);
+      var rowsToEvaluate = rows.Except(evaluationCache.Keys);
+      var rowsEnumerator = rowsToEvaluate.GetEnumerator();
+      var valuesEnumerator = (from xs in GetEstimatedValueVectors(ProblemData.Dataset, rowsToEvaluate)
+                              select AggregateEstimatedValues(xs))
+                             .GetEnumerator();
+
+      while (rowsEnumerator.MoveNext() & valuesEnumerator.MoveNext()) {
+        evaluationCache.Add(rowsEnumerator.Current, valuesEnumerator.Current);
+      }
+
+      return rows.Select(row => evaluationCache[row]);
     }
 
@@ -223 +249 @@
 
     protected override void OnProblemDataChanged() {
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
       IRegressionProblemData problemData = new RegressionProblemData(ProblemData.Dataset,
                                                                      ProblemData.AllowedInputVariables,
@@ -251 +280 @@
     public void AddRegressionSolutions(IEnumerable<IRegressionSolution> solutions) {
       regressionSolutions.AddRange(solutions);
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
     public void RemoveRegressionSolutions(IEnumerable<IRegressionSolution> solutions) {
       regressionSolutions.RemoveRange(solutions);
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
 
@@ -275 +312 @@
       trainingPartitions[solution.Model] = solution.ProblemData.TrainingPartition;
       testPartitions[solution.Model] = solution.ProblemData.TestPartition;
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
 
@@ -282 +323 @@
       trainingPartitions.Remove(solution.Model);
       testPartitions.Remove(solution.Model);
+
+      trainingEvaluationCache.Clear();
+      testEvaluationCache.Clear();
+      evaluationCache.Clear();
     }
   }

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionProblemData.cs

@@ -95 +95 @@
     #endregion
 
-    public ConstrainedValueParameter<StringValue> TargetVariableParameter {
-      get { return (ConstrainedValueParameter<StringValue>)Parameters[TargetVariableParameterName]; }
+    public IConstrainedValueParameter<StringValue> TargetVariableParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[TargetVariableParameterName]; }
     }
     public string TargetVariable {

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolution.cs

@@ -55 +55 @@
     }
     public override IEnumerable<double> EstimatedTrainingValues {
-      get { return GetEstimatedValues(ProblemData.TrainingIndizes); }
+      get { return GetEstimatedValues(ProblemData.TrainingIndices); }
     }
     public override IEnumerable<double> EstimatedTestValues {
-      get { return GetEstimatedValues(ProblemData.TestIndizes); }
+      get { return GetEstimatedValues(ProblemData.TestIndices); }
     }
 

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/Regression/RegressionSolutionBase.cs

@@ -138 +138 @@
         OnlineCalculatorError errorState;
         Add(new Result(TrainingMeanAbsoluteErrorResultName, "Mean of absolute errors of the model on the training partition", new DoubleValue()));
-        double trainingMAE = OnlineMeanAbsoluteErrorCalculator.Calculate(EstimatedTrainingValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndizes), out errorState);
+        double trainingMAE = OnlineMeanAbsoluteErrorCalculator.Calculate(EstimatedTrainingValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices), out errorState);
         TrainingMeanAbsoluteError = errorState == OnlineCalculatorError.None ? trainingMAE : double.NaN;
       }
@@ -145 +145 @@
         OnlineCalculatorError errorState;
         Add(new Result(TestMeanAbsoluteErrorResultName, "Mean of absolute errors of the model on the test partition", new DoubleValue()));
-        double testMAE = OnlineMeanAbsoluteErrorCalculator.Calculate(EstimatedTestValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndizes), out errorState);
+        double testMAE = OnlineMeanAbsoluteErrorCalculator.Calculate(EstimatedTestValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices), out errorState);
         TestMeanAbsoluteError = errorState == OnlineCalculatorError.None ? testMAE : double.NaN;
       }
@@ -152 +152 @@
         OnlineCalculatorError errorState;
         Add(new Result(TrainingMeanErrorResultName, "Mean of errors of the model on the training partition", new DoubleValue()));
-        double trainingME = OnlineMeanErrorCalculator.Calculate(EstimatedTrainingValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndizes), out errorState);
+        double trainingME = OnlineMeanErrorCalculator.Calculate(EstimatedTrainingValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices), out errorState);
         TrainingMeanError = errorState == OnlineCalculatorError.None ? trainingME : double.NaN;
       }
@@ -158 +158 @@
         OnlineCalculatorError errorState;
         Add(new Result(TestMeanErrorResultName, "Mean of errors of the model on the test partition", new DoubleValue()));
-        double testME = OnlineMeanErrorCalculator.Calculate(EstimatedTestValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndizes), out errorState);
+        double testME = OnlineMeanErrorCalculator.Calculate(EstimatedTestValues, ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices), out errorState);
         TestMeanError = errorState == OnlineCalculatorError.None ? testME : double.NaN;
       }
@@ -166 +166 @@
     protected void CalculateResults() {
       IEnumerable<double> estimatedTrainingValues = EstimatedTrainingValues; // cache values
-      IEnumerable<double> originalTrainingValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndizes);
+      IEnumerable<double> originalTrainingValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices);
       IEnumerable<double> estimatedTestValues = EstimatedTestValues; // cache values
-      IEnumerable<double> originalTestValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndizes);
+      IEnumerable<double> originalTestValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices);
 
       OnlineCalculatorError errorState;

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/IDataAnalysisProblemData.cs

@@ -36 +36 @@
     IntRange TestPartition { get; }
 
-    IEnumerable<int> TrainingIndizes { get; }
-    IEnumerable<int> TestIndizes { get; }
+    IEnumerable<int> TrainingIndices { get; }
+    IEnumerable<int> TestIndices { get; }
 
     bool IsTrainingSample(int index);

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/Interfaces/IOnlineCalculator.cs

@@ -24 +24 @@
 namespace HeuristicLab.Problems.DataAnalysis {
   [Flags]
-  public enum OnlineCalculatorError { 
+  public enum OnlineCalculatorError {
     /// <summary>
     /// No error occurred
     /// </summary>
-    None = 0, 
+    None = 0,
     /// <summary>
     /// An invalid value has been added (often +/- Infinity and NaN are invalid values)
     /// </summary>
-    InvalidValueAdded = 1, 
+    InvalidValueAdded = 1,
     /// <summary>
     /// The number of elements added to the evaluator is not sufficient to calculate the result value

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/HoeffdingsDependenceCalculator.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
-using HeuristicLab.Common;
 
 namespace HeuristicLab.Problems.DataAnalysis {

branches/DatasetFeatureCorrelation/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/OnlineLinearScalingParameterCalculator.cs

@@ -55 +55 @@
     }
 
-    private int cnt;
     private OnlineMeanAndVarianceCalculator targetMeanCalculator;
     private OnlineMeanAndVarianceCalculator originalMeanAndVarianceCalculator;
@@ -68 +67 @@
 
     public void Reset() {
-      cnt = 0;
       targetMeanCalculator.Reset();
       originalMeanAndVarianceCalculator.Reset();
@@ -85 +83 @@
       originalTargetCovarianceCalculator.Add(original, target);
 
-      cnt++;
     }