Changeset 6760 for branches/PersistenceSpeedUp/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

Timestamp:

09/14/11 13:59:25 (13 years ago)

Author:

epitzer

Message:

#1530 integrate changes from trunk

Location:

branches/PersistenceSpeedUp

Files:

• . (modified) (2 props)
• HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs (modified) (9 diffs)

branches/PersistenceSpeedUp

@@ -12 +12 @@
 *.psess
 *.vsp
+*.docstates

@@ -12 +12 @@
 *.psess
 *.vsp
+*.docstates

branches/PersistenceSpeedUp/HeuristicLab.Algorithms.DataAnalysis/3.4/CrossValidation.cs

@@ -34 +34 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  [Item("Cross Validation", "Cross Validation wrapper for data analysis algorithms.")]
+  [Item("Cross Validation", "Cross-validation wrapper for data analysis algorithms.")]
   [Creatable("Data Analysis")]
   [StorableClass]
@@ -363 +363 @@
 
     public void CollectResultValues(IDictionary<string, IItem> results) {
+      var clonedResults = (ResultCollection)this.results.Clone();
+      foreach (var result in clonedResults) {
+        results.Add(result.Name, result.Value);
+      }
+    }
+
+    private void AggregateResultValues(IDictionary<string, IItem> results) {
       Dictionary<string, List<double>> resultValues = new Dictionary<string, List<double>>();
       IEnumerable<IRun> runs = clonedAlgorithms.Select(alg => alg.Runs.FirstOrDefault()).Where(run => run != null);
@@ -374 +381 @@
         results.Add(result.Name, result.Value);
       foreach (IResult result in ExtractAndAggregateRegressionSolutions(resultCollections)) {
+        results.Add(result.Name, result.Value);
+      }
+      foreach (IResult result in ExtractAndAggregateClassificationSolutions(resultCollections)) {
         results.Add(result.Name, result.Value);
       }
@@ -394 +404 @@
       List<IResult> aggregatedResults = new List<IResult>();
       foreach (KeyValuePair<string, List<IRegressionSolution>> solutions in resultSolutions) {
-        var problemDataClone = (IRegressionProblemData)Problem.ProblemData.Clone();
+        // clone manually to correctly clone references between cloned root objects 
+        Cloner cloner = new Cloner();
+        var problemDataClone = (IRegressionProblemData)cloner.Clone(Problem.ProblemData);
+        // set partitions of problem data clone correctly
         problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
         problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
-        var ensembleSolution = new RegressionEnsembleSolution(solutions.Value.Select(x => x.Model), problemDataClone,
-          solutions.Value.Select(x => x.ProblemData.TrainingPartition),
-          solutions.Value.Select(x => x.ProblemData.TestPartition));
-
-        aggregatedResults.Add(new Result(solutions.Key, ensembleSolution));
-      }
-      return aggregatedResults;
+        // clone models
+        var ensembleSolution = new RegressionEnsembleSolution(
+          solutions.Value.Select(x => cloner.Clone(x.Model)),
+          problemDataClone,
+          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TrainingPartition)),
+          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TestPartition)));
+
+        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+      }
+      List<IResult> flattenedResults = new List<IResult>();
+      CollectResultsRecursively("", aggregatedResults, flattenedResults);
+      return flattenedResults;
+    }
+
+    private IEnumerable<IResult> ExtractAndAggregateClassificationSolutions(IEnumerable<KeyValuePair<string, IItem>> resultCollections) {
+      Dictionary<string, List<IClassificationSolution>> resultSolutions = new Dictionary<string, List<IClassificationSolution>>();
+      foreach (var result in resultCollections) {
+        var classificationSolution = result.Value as IClassificationSolution;
+        if (classificationSolution != null) {
+          if (resultSolutions.ContainsKey(result.Key)) {
+            resultSolutions[result.Key].Add(classificationSolution);
+          } else {
+            resultSolutions.Add(result.Key, new List<IClassificationSolution>() { classificationSolution });
+          }
+        }
+      }
+      var aggregatedResults = new List<IResult>();
+      foreach (KeyValuePair<string, List<IClassificationSolution>> solutions in resultSolutions) {
+        // clone manually to correctly clone references between cloned root objects 
+        Cloner cloner = new Cloner();
+        var problemDataClone = (IClassificationProblemData)cloner.Clone(Problem.ProblemData);
+        // set partitions of problem data clone correctly
+        problemDataClone.TrainingPartition.Start = SamplesStart.Value; problemDataClone.TrainingPartition.End = SamplesEnd.Value;
+        problemDataClone.TestPartition.Start = SamplesStart.Value; problemDataClone.TestPartition.End = SamplesEnd.Value;
+        // clone models
+        var ensembleSolution = new ClassificationEnsembleSolution(
+          solutions.Value.Select(x => cloner.Clone(x.Model)),
+          problemDataClone,
+          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TrainingPartition)),
+          solutions.Value.Select(x => cloner.Clone(x.ProblemData.TestPartition)));
+
+        aggregatedResults.Add(new Result(solutions.Key + " (ensemble)", ensembleSolution));
+      }
+      List<IResult> flattenedResults = new List<IResult>();
+      CollectResultsRecursively("", aggregatedResults, flattenedResults);
+      return flattenedResults;
+    }
+
+    private void CollectResultsRecursively(string path, IEnumerable<IResult> results, IList<IResult> flattenedResults) {
+      foreach (IResult result in results) {
+        flattenedResults.Add(new Result(path + result.Name, result.Value));
+        ResultCollection childCollection = result.Value as ResultCollection;
+        if (childCollection != null) {
+          CollectResultsRecursively(path + result.Name + ".", childCollection, flattenedResults);
+        }
+      }
     }
 
@@ -428 +490 @@
       foreach (KeyValuePair<string, List<double>> resultValue in resultValues) {
         doubleValue.Value = resultValue.Value.Average();
-        aggregatedResults.Add(new Result(resultValue.Key, (IItem)doubleValue.Clone()));
+        aggregatedResults.Add(new Result(resultValue.Key + " (average)", (IItem)doubleValue.Clone()));
         doubleValue.Value = resultValue.Value.StandardDeviation();
-        aggregatedResults.Add(new Result(resultValue.Key + " StdDev", (IItem)doubleValue.Clone()));
+        aggregatedResults.Add(new Result(resultValue.Key + " (std.dev.)", (IItem)doubleValue.Clone()));
       }
       return aggregatedResults;
@@ -481 +543 @@
         throw new ArgumentException("A cross validation algorithm can only contain DataAnalysisProblems.");
       }
+      algorithm.Problem.Reset += (x,y) => OnProblemChanged();
       problem = (IDataAnalysisProblem)algorithm.Problem;
       OnProblemChanged();
@@ -491 +554 @@
       SamplesStart.Value = 0;
       if (Problem != null) {
-        Problem.ProblemDataChanged += (object sender, EventArgs e) => OnProblemChanged();
         SamplesEnd.Value = Problem.ProblemData.Dataset.Rows;
 
@@ -510 +572 @@
       } else
         SamplesEnd.Value = 0;
+
+      SamplesStart_ValueChanged(this, EventArgs.Empty);
+      SamplesEnd_ValueChanged(this, EventArgs.Empty);
     }
 
@@ -656 +721 @@
       stopPending = false;
       Dictionary<string, IItem> collectedResults = new Dictionary<string, IItem>();
-      CollectResultValues(collectedResults);
+      AggregateResultValues(collectedResults);
       results.AddRange(collectedResults.Select(x => new Result(x.Key, x.Value)).Cast<IResult>().ToArray());
       runsCounter++;

@@ -12 +12 @@
 *.psess
 *.vsp
+*.docstates