Changeset 15002 for trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

Timestamp:

05/30/17 15:03:51 (7 years ago)

Author:

bburlacu

Message:

#2760: Got rid of the shuffledProblemData by using a shared seed for all the folds (so that the dataset for each fold is shuffled in exactly the same way). Backwards compatibility should be restored.

File:

• trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs (modified) (10 diffs)

trunk/sources/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

@@ -40 +40 @@
   [StorableClass]
   public sealed class CrossValidation : ParameterizedNamedItem, IAlgorithm, IStorableContent {
-    private IDataAnalysisProblemData shuffledProblemData;
+    [Storable]
+    private int seed;
 
     public CrossValidation()
@@ -94 +95 @@
       samplesEnd = cloner.Clone(original.samplesEnd);
       shuffleSamples = cloner.Clone(original.shuffleSamples);
+      seed = original.seed;
+
       RegisterEvents();
       if (Algorithm != null) RegisterAlgorithmEvents();
@@ -279 +282 @@
         throw new InvalidOperationException(string.Format("Start not allowed in execution state \"{0}\".", ExecutionState));
 
+      seed = new FastRandom().NextInt();
+
       if (Algorithm != null) {
         //create cloned algorithms
@@ -287 +292 @@
             var cloner = new Cloner();
             if (ShuffleSamples.Value) {
+              var random = new FastRandom(seed);
               var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
               var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
-              shuffledDataset = shuffledDataset ?? dataset.Shuffle(new FastRandom());
+              shuffledDataset = shuffledDataset ?? dataset.Shuffle(random);
               cloner.RegisterClonedObject(dataset, shuffledDataset);
             }
@@ -316 +322 @@
             clonedAlgorithm.Prepare();
             clonedAlgorithms.Add(clonedAlgorithm);
-          }
-          // save the shuffled problem data because it is necessary when creating the ensemble solution
-          if (shuffledProblemData == null && shuffledDataset != null) {
-            var dataAnalysisProblem = (IDataAnalysisProblem)algorithm.Problem;
-            var dataset = (Dataset)dataAnalysisProblem.ProblemData.Dataset;
-            var cloner = new Cloner();
-            cloner.RegisterClonedObject(dataset, shuffledDataset);
-            shuffledProblemData = cloner.Clone(dataAnalysisProblem.ProblemData);
           }
         }
@@ -446 +444 @@
         // clone manually to correctly clone references between cloned root objects 
         Cloner cloner = new Cloner();
-        var problemDataClone = ShuffleSamples.Value
-          ? (IRegressionProblemData)cloner.Clone(shuffledProblemData)
-          : (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
+        if (ShuffleSamples.Value) {
+          var dataset = (Dataset)Problem.ProblemData.Dataset;
+          var random = new FastRandom(seed);
+          var shuffledDataset = dataset.Shuffle(random);
+          cloner.RegisterClonedObject(dataset, shuffledDataset);
+        }
+        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
         // set partitions of problem data clone correctly
         problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
@@ -479 +481 @@
         // at least one algorithm (GBT with logistic regression loss) produces a classification solution even though the original problem is a regression problem.
         var targetVariable = solutions.Value.First().ProblemData.TargetVariable;
-        var problemDataClone = ShuffleSamples.Value
-          ? new ClassificationProblemData(shuffledProblemData.Dataset, shuffledProblemData.AllowedInputVariables, targetVariable)
-          : new ClassificationProblemData(Problem.ProblemData.Dataset, Problem.ProblemData.AllowedInputVariables, targetVariable);
+        var dataset = (Dataset)Problem.ProblemData.Dataset;
+        if (ShuffleSamples.Value) {
+          var random = new FastRandom(seed);
+          dataset = dataset.Shuffle(random);
+        }
+        var problemDataClone = new ClassificationProblemData(dataset, Problem.ProblemData.AllowedInputVariables, targetVariable);
         // set partitions of problem data clone correctly
         problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
@@ -566 +571 @@
       if (Problem != null) {
         Problem.Reset += new EventHandler(Problem_Reset);
-        Problem.ProblemDataChanged += Problem_ProblemDataChanged;
       }
     }
@@ -574 +578 @@
       if (Problem != null) {
         Problem.Reset -= new EventHandler(Problem_Reset);
-        Problem.ProblemDataChanged -= Problem_ProblemDataChanged;
       }
     }
@@ -592 +595 @@
       if (handler != null) handler(this, EventArgs.Empty);
       ConfigureProblem();
-    }
-    public event EventHandler ProblemDataChanged;
-    private void OnProblemDataChanged() {
-      var handler = ProblemDataChanged;
-      if (handler != null) handler(this, EventArgs.Empty);
-      shuffledProblemData = null;
-    }
-    private void Problem_ProblemDataChanged(object sender, EventArgs e) {
-      OnProblemDataChanged();
     }
     private void Problem_Reset(object sender, EventArgs e) {