Changeset 12800 for branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

Timestamp:

07/22/15 16:20:26 (9 years ago)

Author:

gkronber

Message:

#2434 added support for a partition variable to CrossValidation

File:

• branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs (modified) (14 diffs)

branches/crossvalidation-2434/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

@@ -52 +52 @@
       results = new ResultCollection();
 
-      folds = new IntValue(2);
+      folds = 2;
       numberOfWorkers = new IntValue(1);
       samplesStart = new IntValue(0);
@@ -73 +73 @@
       RegisterEvents();
       if (Algorithm != null) RegisterAlgorithmEvents();
+
+      // BackwardsCompatibility3.4
+      #region Backwards compatible code, remove with 3.6
+      if (persistenceCompatibilityFolds != null) folds = persistenceCompatibilityFolds.Value;
+      #endregion
     }
 
@@ -85 +90 @@
       results = cloner.Clone(original.results);
 
-      folds = cloner.Clone(original.folds);
+      folds = original.folds;
       numberOfWorkers = cloner.Clone(original.numberOfWorkers);
       samplesStart = cloner.Clone(original.samplesStart);
@@ -171 +176 @@
     }
 
-    [Storable]
-    private IntValue folds;
-    public IntValue Folds {
+    // BackwardsCompatibility3.4
+    #region Backwards compatible code, remove with 3.6
+    [Storable(Name = "folds")]
+    private IntValue persistenceCompatibilityFolds;
+    #endregion
+
+    [Storable(Name = "foldsNew")]
+    private int folds;
+    public int Folds {
       get { return folds; }
-    }
+      set {
+        if (value != folds) {
+          folds = value;
+          partitionVariable = NoPartitionVariable; // setting folds updates the partition variable
+          OnFoldsChanged();
+        }
+      }
+    }
+
+    [Storable]
+    // folds are either specified explicitly or by a varaible from the dataset 
+    public const string NoPartitionVariable = "<none>";
+    private string partitionVariable = NoPartitionVariable;
+    public string PartitionVariable {
+      get { return partitionVariable; }
+      set {
+        if (value != partitionVariable) {
+          partitionVariable = value;
+          UpdateNumberOfFolds();
+        }
+      }
+    }
+
     [Storable]
     private IntValue samplesStart;
@@ -244 +277 @@
       }
     }
+
     #endregion
 
@@ -273 +307 @@
         //create cloned algorithms
         if (clonedAlgorithms.Count == 0) {
-          int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / Folds.Value;
-
-          for (int i = 0; i < Folds.Value; i++) {
+
+          for (int i = 0; i < folds; i++) {
             IAlgorithm clonedAlgorithm = (IAlgorithm)algorithm.Clone();
             clonedAlgorithm.Name = algorithm.Name + " Fold " + i;
@@ -281 +314 @@
             ISymbolicDataAnalysisProblem symbolicProblem = problem as ISymbolicDataAnalysisProblem;
 
-            int testStart = (i * testSamplesCount) + SamplesStart.Value;
-            int testEnd = (i + 1) == Folds.Value ? SamplesEnd.Value : (i + 1) * testSamplesCount + SamplesStart.Value;
+            int trainingStart, trainingEnd, testStart, testEnd;
+            // assumes that partitions are subset of subsequent rows
+            GetTrainingAndTestPartitions(i, out trainingStart, out trainingEnd, out testStart, out testEnd);
 
             problem.ProblemData.TrainingPartition.Start = SamplesStart.Value;
@@ -306 +340 @@
         int startedAlgorithms = 0;
         foreach (IAlgorithm clonedAlgorithm in clonedAlgorithms) {
-          if (startedAlgorithms < NumberOfWorkers.Value) {
+          if (startedAlgorithms < numberOfWorkers.Value) {
             if (clonedAlgorithm.ExecutionState == ExecutionState.Prepared ||
                 clonedAlgorithm.ExecutionState == ExecutionState.Paused) {
@@ -324 +358 @@
         }
         OnStarted();
+      }
+    }
+
+    private void GetTrainingAndTestPartitions(int fold, out int trainingStart, out int trainingEnd, out int testStart, out int testEnd) {
+      if (partitionVariable == NoPartitionVariable) {
+        // uniform split
+        int testSamplesCount = (SamplesEnd.Value - SamplesStart.Value) / folds;
+        trainingStart = SamplesStart.Value;
+        trainingEnd = SamplesEnd.Value;
+        testStart = (fold * testSamplesCount) + SamplesStart.Value;
+        testEnd = (fold + 1) == folds ? SamplesEnd.Value : (fold + 1) * testSamplesCount + SamplesStart.Value;
+      } else {
+        // group rowIdx by partition
+        var partition = Problem.ProblemData.Dataset.GetReadOnlyDoubleValues(partitionVariable);
+        var g = Enumerable.Range(SamplesStart.Value, SamplesEnd.Value - SamplesStart.Value).GroupBy(r => partition[r]).OrderBy(r => r.Key).ToList();
+        trainingStart = SamplesStart.Value;
+        trainingEnd = SamplesEnd.Value;
+        testStart = g[fold].Min();
+        testEnd = g[fold].Max() + 1;
       }
     }
@@ -365 +418 @@
       values.Add("Algorithm Name", new StringValue(Name));
       values.Add("Algorithm Type", new StringValue(GetType().GetPrettyName()));
-      values.Add("Folds", new IntValue(Folds.Value));
+      values.Add("Folds", new IntValue(folds));
 
       if (algorithm != null) {
@@ -516 +569 @@
     #region events
     private void RegisterEvents() {
-      Folds.ValueChanged += new EventHandler(Folds_ValueChanged);
+      SamplesStart.ValueChanged += (s, e) => UpdateNumberOfFolds();
+      SamplesEnd.ValueChanged += (s, e) => UpdateNumberOfFolds();
+
       RegisterClonedAlgorithmsEvents();
     }
-    private void Folds_ValueChanged(object sender, EventArgs e) {
-      if (ExecutionState != ExecutionState.Prepared)
-        throw new InvalidOperationException("Can not change number of folds if the execution state is not prepared.");
+
+    private void UpdateNumberOfFolds() {
+      if (ExecutionState == ExecutionState.Paused || executionState == ExecutionState.Started)
+        throw new InvalidOperationException("Can not change number of folds if crossvalidation is paused or started.");
+
+      if (partitionVariable != NoPartitionVariable) {
+        // number of folds is the number of distinct values of the partition variable in the range [SamplesStart..SamplesEnd[
+        var ds = Problem.ProblemData.Dataset;
+        var partitionValues = ds.GetDoubleValues(PartitionVariable, Enumerable.Range(SamplesStart.Value, SamplesEnd.Value - SamplesStart.Value)).Distinct().Count();
+        folds = partitionValues;
+        OnFoldsChanged();
+      }
     }
 
@@ -555 +619 @@
     public event EventHandler ProblemChanged;
     private void OnProblemChanged() {
+      ConfigureProblem();
       EventHandler handler = ProblemChanged;
       if (handler != null) handler(this, EventArgs.Empty);
-      ConfigureProblem();
     }
 
@@ -565 +629 @@
 
     private void ConfigureProblem() {
+      folds = 2;
+      partitionVariable = NoPartitionVariable;
       SamplesStart.Value = 0;
       if (Problem != null) {
@@ -687 +753 @@
 
     #region event firing
+
+    public event EventHandler FoldsChanged;
+    private void OnFoldsChanged() {
+      EventHandler handler = FoldsChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
     public event EventHandler ExecutionStateChanged;
     private void OnExecutionStateChanged() {