Changeset 4468 for trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3

Timestamp:

09/22/10 11:22:49 (14 years ago)

Author:

mkommend

Message:

Preparation for cross validation - removed the test samples from the trainining samples and added ValidationPercentage parameter (ticket #1199).

Location:

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3

Files:

• Analyzers/RegressionSolutionAnalyzer.cs (modified) (3 diffs)
• Symbolic/Analyzers/BestSymbolicRegressionSolutionAnalyzer.cs (modified) (1 diff)
• Symbolic/Analyzers/FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs (modified) (3 diffs)
• Symbolic/Analyzers/SymbolicRegressionModelQualityAnalyzer.cs (modified) (4 diffs)
• Symbolic/Analyzers/SymbolicRegressionModelQualityCalculator.cs (modified) (2 diffs)
• Symbolic/Analyzers/SymbolicRegressionTournamentPruning.cs (modified) (2 diffs)
• Symbolic/Analyzers/ValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs (modified) (1 diff)
• Symbolic/Evaluators/MultiObjectiveSymbolicRegressionEvaluator.cs (modified) (2 diffs)
• Symbolic/Evaluators/SingleObjectiveSymbolicRegressionEvaluator.cs (modified) (2 diffs)
• Symbolic/SymbolicRegressionModel.cs (modified) (1 diff)
• Symbolic/SymbolicRegressionProblem.cs (modified) (1 diff)
• Symbolic/SymbolicRegressionProblemBase.cs (modified) (2 diffs)
• Symbolic/SymbolicRegressionSolution.cs (modified) (1 diff)

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Analyzers/RegressionSolutionAnalyzer.cs

@@ -121 +121 @@
       var solution = bestSolution;
       #region update R2,MSE, Rel Error
-      IEnumerable<double> trainingValues = problemData.Dataset.GetEnumeratedVariableValues(
-        problemData.TargetVariable.Value,
-        problemData.TrainingSamplesStart.Value,
-        problemData.TrainingSamplesEnd.Value);
-      IEnumerable<double> testValues = problemData.Dataset.GetEnumeratedVariableValues(
-        problemData.TargetVariable.Value,
-        problemData.TestSamplesStart.Value,
-        problemData.TestSamplesEnd.Value);
+      IEnumerable<double> trainingValues = problemData.Dataset.GetEnumeratedVariableValues(problemData.TargetVariable.Value, problemData.TrainingIndizes);
+      IEnumerable<double> testValues = problemData.Dataset.GetEnumeratedVariableValues(problemData.TargetVariable.Value, problemData.TestIndizes);
       OnlineMeanSquaredErrorEvaluator mseEvaluator = new OnlineMeanSquaredErrorEvaluator();
       OnlineMeanAbsolutePercentageErrorEvaluator relErrorEvaluator = new OnlineMeanAbsolutePercentageErrorEvaluator();
       OnlinePearsonsRSquaredEvaluator r2Evaluator = new OnlinePearsonsRSquaredEvaluator();
+
       #region training
       var originalEnumerator = trainingValues.GetEnumerator();
@@ -144 +139 @@
       double trainingRelError = relErrorEvaluator.MeanAbsolutePercentageError;
       #endregion
+
       mseEvaluator.Reset();
       relErrorEvaluator.Reset();
       r2Evaluator.Reset();
+
       #region test
       originalEnumerator = testValues.GetEnumerator();
@@ -159 +156 @@
       double testRelError = relErrorEvaluator.MeanAbsolutePercentageError;
       #endregion
+
       if (results.ContainsKey(BestSolutionResultName)) {
         results[BestSolutionResultName].Value = solution;

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/BestSymbolicRegressionSolutionAnalyzer.cs

@@ -91 +91 @@
         var model = new SymbolicRegressionModel((ISymbolicExpressionTreeInterpreter)SymbolicExpressionTreeInterpreter.Clone(),
           SymbolicExpressionTree[i]);
-        var solution = new SymbolicRegressionSolution(ProblemData, model, lowerEstimationLimit, upperEstimationLimit);
+        DataAnalysisProblemData problemDataClone = (DataAnalysisProblemData)ProblemData.Clone();
+        var solution = new SymbolicRegressionSolution(problemDataClone, model, lowerEstimationLimit, upperEstimationLimit);
         solution.Name = BestSolutionParameterName;
         solution.Description = "Best solution on validation partition found over the whole run.";
         BestSolutionParameter.ActualValue = solution;
         BestSolutionQualityParameter.ActualValue = Quality[i];
-        BestSymbolicRegressionSolutionAnalyzer.UpdateSymbolicRegressionBestSolutionResults(solution, ProblemData, Results, VariableFrequencies);
+        BestSymbolicRegressionSolutionAnalyzer.UpdateSymbolicRegressionBestSolutionResults(solution, problemDataClone, Results, VariableFrequencies);
       }
       return BestSolutionParameter.ActualValue;

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/FixedValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs

@@ -212 +212 @@
       int count = (int)((validationEnd - validationStart) * RelativeNumberOfEvaluatedSamples.Value);
       if (count == 0) count = 1;
-      IEnumerable<int> rows = RandomEnumerable.SampleRandomNumbers(seed, validationStart, validationEnd, count);
+      IEnumerable<int> rows = RandomEnumerable.SampleRandomNumbers(seed, validationStart, validationEnd, count)
+        .Where(row => row < ProblemData.TestSamplesStart.Value || ProblemData.TestSamplesEnd.Value <= row);
 
       double upperEstimationLimit = UpperEstimationLimit != null ? UpperEstimationLimit.Value : double.PositiveInfinity;
@@ -241 +242 @@
         // calculate scaling parameters and only for the best tree using the full training set
         double alpha, beta;
-        int trainingStart = ProblemData.TrainingSamplesStart.Value;
-        int trainingEnd = ProblemData.TrainingSamplesEnd.Value;
-        IEnumerable<int> trainingRows = Enumerable.Range(trainingStart, trainingEnd - trainingStart);
         SymbolicRegressionScaledMeanSquaredErrorEvaluator.Calculate(SymbolicExpressionTreeInterpreter, bestTree,
           lowerEstimationLimit, upperEstimationLimit,
           ProblemData.Dataset, targetVariable,
-          trainingRows, out beta, out alpha);
+          ProblemData.TrainingIndizes, out beta, out alpha);
 
         // scale tree for solution
@@ -253 +251 @@
         var model = new SymbolicRegressionModel((ISymbolicExpressionTreeInterpreter)SymbolicExpressionTreeInterpreter.Clone(),
           scaledTree);
-        var solution = new SymbolicRegressionSolution(ProblemData, model, lowerEstimationLimit, upperEstimationLimit);
+        var solution = new SymbolicRegressionSolution((DataAnalysisProblemData)ProblemData.Clone(), model, lowerEstimationLimit, upperEstimationLimit);
         solution.Name = BestSolutionParameterName;
         solution.Description = "Best solution on validation partition found over the whole run.";

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/SymbolicRegressionModelQualityAnalyzer.cs

@@ -137 +137 @@
       Analyze(SymbolicExpressionTreeParameter.ActualValue, SymbolicExpressionTreeInterpreterParameter.ActualValue,
         UpperEstimationLimit.Value, LowerEstimationLimit.Value, ProblemDataParameter.ActualValue,
-        ProblemDataParameter.ActualValue.TrainingSamplesStart.Value, ProblemDataParameter.ActualValue.TrainingSamplesEnd.Value,
-        ProblemDataParameter.ActualValue.TestSamplesStart.Value, ProblemDataParameter.ActualValue.TestSamplesEnd.Value,
         ResultsParameter.ActualValue);
       return base.Apply();
@@ -145 +143 @@
     public static void Analyze(IEnumerable<SymbolicExpressionTree> trees, ISymbolicExpressionTreeInterpreter interpreter,
       double upperEstimationLimit, double lowerEstimationLimit,
-      DataAnalysisProblemData problemData, int trainingStart, int trainingEnd, int testStart, int testEnd, ResultCollection results) {
+      DataAnalysisProblemData problemData, ResultCollection results) {
       int targetVariableIndex = problemData.Dataset.GetVariableIndex(problemData.TargetVariable.Value);
-      IEnumerable<double> originalTrainingValues = problemData.Dataset.GetEnumeratedVariableValues(targetVariableIndex, trainingStart, trainingEnd);
-      IEnumerable<double> originalTestValues = problemData.Dataset.GetEnumeratedVariableValues(targetVariableIndex, testStart, testEnd);
+      IEnumerable<double> originalTrainingValues = problemData.Dataset.GetEnumeratedVariableValues(targetVariableIndex, problemData.TrainingIndizes);
+      IEnumerable<double> originalTestValues = problemData.Dataset.GetEnumeratedVariableValues(targetVariableIndex, problemData.TestIndizes);
       List<double> trainingMse = new List<double>();
       List<double> trainingR2 = new List<double>();
@@ -162 +160 @@
       foreach (var tree in trees) {
         #region training
-        var estimatedTrainingValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, Enumerable.Range(trainingStart, trainingEnd - trainingStart));
+        var estimatedTrainingValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, problemData.TrainingIndizes);
         mseEvaluator.Reset();
         r2Evaluator.Reset();
@@ -184 +182 @@
         #endregion
         #region test
-        var estimatedTestValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, Enumerable.Range(testStart, testEnd - testStart));
+        var estimatedTestValues = interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, problemData.TestIndizes);
 
         mseEvaluator.Reset();

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/SymbolicRegressionModelQualityCalculator.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
@@ -35 +36 @@
   [Item("SymbolicRegressionModelQualityCalculator", "An operator to calculate the quality values of a symbolic regression solution symbolic expression tree encoding.")]
   [StorableClass]
+  [Obsolete("This class should not be used anymore because of performance reasons and will therefore not be updated.")]
   public sealed class SymbolicRegressionModelQualityCalculator : AlgorithmOperator {
     private const string SymbolicExpressionTreeInterpreterParameterName = "SymbolicExpressionTreeInterpreter";

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/SymbolicRegressionTournamentPruning.cs

@@ -28 +28 @@
 using HeuristicLab.Optimization;
 using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
 using HeuristicLab.Problems.DataAnalysis.Symbolic.Symbols;
-using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 
 namespace HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Analyzers {
@@ -241 +241 @@
       double lowerEstimationLimit, double upperEstimationLimit,
       double maxPruningRatio, double qualityGainWeight) {
-      IEnumerable<int> rows = Enumerable.Range(samplesStart, samplesEnd - samplesStart);
+        IEnumerable<int> rows = Enumerable.Range(samplesStart, samplesEnd - samplesStart)
+          .Where(i => i < problemData.TestSamplesStart.Value || problemData.TestSamplesEnd.Value <= i);
       int originalSize = tree.Size;
       double originalQuality = evaluator.Evaluate(interpreter, tree,

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/ValidationBestScaledSymbolicRegressionSolutionAnalyzer.cs

@@ -39 +39 @@
   [Item("ValidationBestScaledSymbolicRegressionSolutionAnalyzer", "An operator that analyzes the validation best scaled symbolic regression solution.")]
   [StorableClass]
+  [Obsolete("This class should not be used anymore because of performance reasons and will therefore not be updated.")]
   public sealed class ValidationBestScaledSymbolicRegressionSolutionAnalyzer : AlgorithmOperator, ISymbolicRegressionAnalyzer {
     private const string SymbolicExpressionTreeParameterName = "SymbolicExpressionTree";

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Evaluators/MultiObjectiveSymbolicRegressionEvaluator.cs

@@ -20 +20 @@
 #endregion
 
-using System;
 using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
@@ -122 +122 @@
     public override IOperation Apply() {
       int seed = Random.Next();
-      IEnumerable<int> rows = SingleObjectiveSymbolicRegressionEvaluator.GenerateRowsToEvaluate(seed, RelativeNumberOfEvaluatedSamples.Value, SamplesStart.Value, SamplesEnd.Value);
+      IEnumerable<int> rows = SingleObjectiveSymbolicRegressionEvaluator.GenerateRowsToEvaluate(seed, RelativeNumberOfEvaluatedSamples.Value, SamplesStart.Value, SamplesEnd.Value)
+         .Where(i => i < RegressionProblemData.TestSamplesStart.Value || RegressionProblemData.TestSamplesEnd.Value <= i);
       double[] qualities = Evaluate(SymbolicExpressionTreeInterpreter, SymbolicExpressionTree, RegressionProblemData.Dataset,
         RegressionProblemData.TargetVariable, rows);

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Evaluators/SingleObjectiveSymbolicRegressionEvaluator.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Core;
 using HeuristicLab.Data;
@@ -140 +141 @@
     public override IOperation Apply() {
       int seed = Random.Next();
-      IEnumerable<int> rows = GenerateRowsToEvaluate(seed, RelativeNumberOfEvaluatedSamples.Value, SamplesStart.Value, SamplesEnd.Value);
+      IEnumerable<int> rows = GenerateRowsToEvaluate(seed, RelativeNumberOfEvaluatedSamples.Value, SamplesStart.Value, SamplesEnd.Value)
+          .Where(i => i < RegressionProblemData.TestSamplesStart.Value || RegressionProblemData.TestSamplesEnd.Value <= i);
       double quality = Evaluate(SymbolicExpressionTreeInterpreter, SymbolicExpressionTree, LowerEstimationLimit.Value, UpperEstimationLimit.Value,
         RegressionProblemData.Dataset,

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/SymbolicRegressionModel.cs

@@ -68 +68 @@
 
     public IEnumerable<double> GetEstimatedValues(DataAnalysisProblemData problemData, int start, int end) {
-      return interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, Enumerable.Range(start, end - start));
+      return GetEstimatedValues(problemData, Enumerable.Range(start, end - start));
+    }
+    public IEnumerable<double> GetEstimatedValues(DataAnalysisProblemData problemData, IEnumerable<int> rows) {
+      return interpreter.GetSymbolicExpressionTreeValues(tree, problemData.Dataset, rows);
     }
 

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/SymbolicRegressionProblem.cs

@@ -170 +170 @@
           fixedBestValidationSolutionAnalyzer.BestKnownQualityParameter.ActualName = BestKnownQualityParameter.Name;
         }
+
         var bestValidationSolutionAnalyzer = analyzer as ValidationBestScaledSymbolicRegressionSolutionAnalyzer;
         if (bestValidationSolutionAnalyzer != null) {

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/SymbolicRegressionProblemBase.cs

@@ -125 +125 @@
     }
     public IntValue TrainingSamplesStart {
-      get { return new IntValue(DataAnalysisProblemData.TrainingSamplesStart.Value); }
+      get { return new IntValue(DataAnalysisProblemData.TrainingIndizes.First()); }
     }
     public IntValue TrainingSamplesEnd {
       get {
-        return new IntValue((DataAnalysisProblemData.TrainingSamplesStart.Value +
-          DataAnalysisProblemData.TrainingSamplesEnd.Value) / 2);
+        int endIndex = (int)(DataAnalysisProblemData.TrainingIndizes.Count() * (1.0 - DataAnalysisProblemData.ValidationPercentage.Value));
+        return new IntValue(DataAnalysisProblemData.TrainingIndizes.ElementAt(endIndex));
       }
     }
@@ -137 +137 @@
     }
     public IntValue ValidationSamplesEnd {
-      get { return new IntValue(DataAnalysisProblemData.TrainingSamplesEnd.Value); }
+      get { return new IntValue(DataAnalysisProblemData.TrainingIndizes.Last() + 1); }
     }
     public IntValue TestSamplesStart {

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/SymbolicRegressionSolution.cs

@@ -67 +67 @@
       get {
         if (estimatedValues == null) RecalculateEstimatedValues();
-        return estimatedValues.AsEnumerable();
+        return estimatedValues;
       }
     }
 
     public override IEnumerable<double> EstimatedTrainingValues {
-      get {
-        if (estimatedValues == null) RecalculateEstimatedValues();
-        int start = ProblemData.TrainingSamplesStart.Value;
-        int n = ProblemData.TrainingSamplesEnd.Value - start;
-        return estimatedValues.Skip(start).Take(n).ToList();
-      }
+      get { return GetEstimatedValues(ProblemData.TrainingIndizes); }
     }
 
     public override IEnumerable<double> EstimatedTestValues {
-      get {
-        if (estimatedValues == null) RecalculateEstimatedValues();
-        int start = ProblemData.TestSamplesStart.Value;
-        int n = ProblemData.TestSamplesEnd.Value - start;
-        return estimatedValues.Skip(start).Take(n).ToList();
-      }
+      get { return GetEstimatedValues(ProblemData.TestIndizes); }
+    }
+
+    public virtual IEnumerable<double> GetEstimatedValues(IEnumerable<int> rows) {
+      if (estimatedValues == null) RecalculateEstimatedValues();
+      foreach (int row in rows)
+        yield return estimatedValues[row];
     }
   }