Changeset 3452 for trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic

Timestamp:

04/20/10 20:31:23 (14 years ago)

Author:

gkronber

Message:

Included tracking of best of run solution (based on validation set) and calculation of MSE, R² and rel. Error on training and test sets. #938 (Data types and operators for regression problems)

Location:

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic

Files:

• BestValidationSymbolicRegressionSolutionVisualizer.cs (modified) (5 diffs)
• ISymbolicRegressionEvaluator.cs (modified) (1 diff)
• SymbolicRegressionEvaluator.cs (modified) (4 diffs)
• SymbolicRegressionProblem.cs (modified) (8 diffs)

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/BestValidationSymbolicRegressionSolutionVisualizer.cs

@@ -29 +29 @@
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Problems.DataAnalysis.Evaluators;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
 
 namespace HeuristicLab.Problems.DataAnalysis.Regression.Symbolic {
@@ -37 +39 @@
   [StorableClass]
   public sealed class BestValidationSymbolicRegressionSolutionVisualizer : SingleSuccessorOperator, ISingleObjectiveSolutionsVisualizer, ISolutionsVisualizer {
+    private const string EvaluatorParameterName = "Evaluator";
     private const string SymbolicRegressionModelParameterName = "SymbolicRegressionModel";
     private const string DataAnalysisProblemDataParameterName = "DataAnalysisProblemData";
     private const string BestValidationSolutionParameterName = "BestValidationSolution";
+    private const string ValidationSamplesStartParameterName = "ValidationSamplesStart";
+    private const string ValidationSamplesEndParameterName = "ValidationSamplesEnd";
     private const string QualityParameterName = "Quality";
+    private const string ResultsParameterName = "Results";
+
+    #region parameter properties
+    public ILookupParameter<ISymbolicRegressionEvaluator> EvaluatorParameter {
+      get { return (ILookupParameter<ISymbolicRegressionEvaluator>)Parameters[EvaluatorParameterName]; }
+    }
+    public IValueLookupParameter<IntValue> ValidationSamplesStartParameter {
+      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationSamplesStartParameterName]; }
+    }
+    public IValueLookupParameter<IntValue> ValidationSamplesEndParameter {
+      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationSamplesEndParameterName]; }
+    }
+
     public ILookupParameter<ItemArray<SymbolicExpressionTree>> SymbolicExpressionTreeParameter {
       get { return (ILookupParameter<ItemArray<SymbolicExpressionTree>>)Parameters[SymbolicRegressionModelParameterName]; }
@@ -58 +76 @@
     }
 
+    public ILookupParameter<ResultCollection> ResultParameter {
+      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public ISymbolicRegressionEvaluator Evaluator {
+      get { return EvaluatorParameter.ActualValue; }
+    }
+    public IntValue ValidationSamplesStart {
+      get { return ValidationSamplesStartParameter.ActualValue; }
+    }
+    public IntValue ValidationSamplesEnd {
+      get { return ValidationSamplesEndParameter.ActualValue; }
+    }
+    #endregion
+
     public BestValidationSymbolicRegressionSolutionVisualizer()
       : base() {
@@ -63 +98 @@
       Parameters.Add(new SubScopesLookupParameter<DoubleValue>(QualityParameterName, "The quality of the symbolic regression solutions."));
       Parameters.Add(new LookupParameter<DataAnalysisProblemData>(DataAnalysisProblemDataParameterName, "The symbolic regression problme data on which the best solution should be evaluated."));
+      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesStartParameterName, "The start index of the validation partition (part of the training partition)."));
+      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesEndParameterName, "The end index of the validation partition (part of the training partition)."));
       Parameters.Add(new LookupParameter<SymbolicRegressionSolution>(BestValidationSolutionParameterName, "The best symbolic expression tree based on the validation data for the symbolic regression problem."));
-      Parameters.Add(new LookupParameter<ResultCollection>("Results"));
+      Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
     }
 
@@ -70 +107 @@
       ItemArray<SymbolicExpressionTree> expressions = SymbolicExpressionTreeParameter.ActualValue;
       DataAnalysisProblemData problemData = DataAnalysisProblemDataParameter.ActualValue;
-      ItemArray<DoubleValue> qualities = QualityParameter.ActualValue;
 
-      var bestExpressionIndex = (from index in Enumerable.Range(0, qualities.Count())
-                                 select new { Index = index, Quality = qualities[index] }).OrderBy(x => x.Quality).Select(x => x.Index).First();
+      int validationSamplesStart = ValidationSamplesStart.Value;
+      int validationSamplesEnd = ValidationSamplesEnd.Value;
+      var validationValues = problemData.Dataset.GetVariableValues(problemData.TargetVariable.Value, validationSamplesStart, validationSamplesEnd);
 
-      var bestExpression = expressions[bestExpressionIndex];
-      SymbolicRegressionSolution bestSolution = BestValidationSolutionParameter.ActualValue;
-      if (bestSolution == null) BestValidationSolutionParameter.ActualValue = CreateDataAnalysisSolution(problemData, bestExpression);
-      else {
-        bestSolution.Model = CreateModel(problemData, bestExpression);
+      var currentBestExpression = (from expression in expressions
+                                   let validationQuality = SymbolicRegressionMeanSquaredErrorEvaluator.Calculate(expression, problemData.Dataset, problemData.TargetVariable.Value, validationSamplesStart, validationSamplesEnd)
+                                   select new { Expression = expression, ValidationQuality = validationQuality })
+                                   .OrderBy(x => x.ValidationQuality)
+                                   .First();
+
+      SymbolicRegressionSolution bestOfRunSolution = BestValidationSolutionParameter.ActualValue;
+      if (bestOfRunSolution == null) {
+        // no best of run solution yet -> make a solution from the currentBestExpression
+        UpdateBestOfRunSolution(problemData, currentBestExpression.Expression);
+      } else {
+        // compare quality of current best with best of run solution
+        var estimatedValidationValues = bestOfRunSolution.EstimatedValues.Skip(validationSamplesStart).Take(validationSamplesEnd - validationSamplesStart);
+        var bestOfRunValidationQuality = SimpleMSEEvaluator.Calculate(validationValues, estimatedValidationValues);
+        if (bestOfRunValidationQuality > currentBestExpression.ValidationQuality) {
+          UpdateBestOfRunSolution(problemData, currentBestExpression.Expression);
+        }
       }
-      // ((ResultCollection)Parameters["Results"].ActualValue).Add(new Result("ValidationMSE", new DoubleValue(3.15)));
+
+
       return base.Apply();
+    }
+
+    private void UpdateBestOfRunSolution(DataAnalysisProblemData problemData, SymbolicExpressionTree tree) {
+      var newBestSolution = CreateDataAnalysisSolution(problemData, tree);
+      BestValidationSolutionParameter.ActualValue = newBestSolution;
+
+      var trainingValues = problemData.Dataset.GetVariableValues(problemData.TargetVariable.Value, problemData.TrainingSamplesStart.Value, problemData.TrainingSamplesEnd.Value);
+      var testValues = problemData.Dataset.GetVariableValues(problemData.TargetVariable.Value, problemData.TestSamplesStart.Value, problemData.TestSamplesEnd.Value);
+
+      AddResult("MeanSquaredError (Training)", new DoubleValue(SimpleMSEEvaluator.Calculate(trainingValues, newBestSolution.EstimatedTrainingValues)));
+      AddResult("MeanRelativeError (Training)", new PercentValue(SimpleMeanAbsolutePercentageErrorEvaluator.Calculate(trainingValues, newBestSolution.EstimatedTrainingValues)));
+      AddResult("RSquared (Training)", new DoubleValue(SimpleRSquaredEvaluator.Calculate(trainingValues, newBestSolution.EstimatedTrainingValues)));
+
+      AddResult("MeanSquaredError (Test)", new DoubleValue(SimpleMSEEvaluator.Calculate(testValues, newBestSolution.EstimatedTestValues)));
+      AddResult("MeanRelativeError (Test)", new PercentValue(SimpleMeanAbsolutePercentageErrorEvaluator.Calculate(testValues, newBestSolution.EstimatedTestValues)));
+      AddResult("RSquared (Test)", new DoubleValue(SimpleRSquaredEvaluator.Calculate(testValues, newBestSolution.EstimatedTestValues)));
+    }
+
+    private void AddResult(string resultName, IItem value) {
+      var resultCollection = ResultParameter.ActualValue;
+      if (resultCollection.ContainsKey(resultName)) {
+        resultCollection[resultName].Value = value;
+      } else {
+        resultCollection.Add(new Result(resultName, value));
+      }
     }
 

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/ISymbolicRegressionEvaluator.cs

@@ -36 +36 @@
 namespace HeuristicLab.Problems.DataAnalysis.Regression.Symbolic {
   public interface ISymbolicRegressionEvaluator : ISingleObjectiveEvaluator {
-    ILookupParameter<SymbolicExpressionTree> FunctionTreeParameter { get; }
+    ILookupParameter<SymbolicExpressionTree> SymbolicExpressionTreeParameter { get; }
     ILookupParameter<DataAnalysisProblemData> RegressionProblemDataParameter { get; }
+    IValueLookupParameter<IntValue> SamplesStartParameter { get; }
+    IValueLookupParameter<IntValue> SamplesEndParameter { get; }
     ILookupParameter<DoubleValue> NumberOfEvaluatedNodesParameter { get; }
   }

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/SymbolicRegressionEvaluator.cs

@@ -42 +42 @@
     private const string FunctionTreeParameterName = "FunctionTree";
     private const string RegressionProblemDataParameterName = "RegressionProblemData";
+    private const string SamplesStartParameterName = "SamplesStart";
+    private const string SamplesEndParameterName = "SamplesEnd";
     private const string NumberOfEvaluatedNodexParameterName = "NumberOfEvaluatedNodes";
     #region ISymbolicRegressionEvaluator Members
@@ -49 +51 @@
     }
 
-    public ILookupParameter<SymbolicExpressionTree> FunctionTreeParameter {
+    public ILookupParameter<SymbolicExpressionTree> SymbolicExpressionTreeParameter {
       get { return (ILookupParameter<SymbolicExpressionTree>)Parameters[FunctionTreeParameterName]; }
     }
@@ -57 +59 @@
     }
 
-    //public ILookupParameter<IntValue> SamplesStartParameter {
-    //  get { return (ILookupParameter<IntValue>)Parameters["SamplesStart"]; }
-    //}
+    public IValueLookupParameter<IntValue> SamplesStartParameter {
+      get { return (IValueLookupParameter<IntValue>)Parameters[SamplesStartParameterName]; }
+    }
 
-    //public ILookupParameter<IntValue> SamplesEndParameter {
-    //  get { return (ILookupParameter<IntValue>)Parameters["SamplesEnd"]; }
-    //}
+    public IValueLookupParameter<IntValue> SamplesEndParameter {
+      get { return (IValueLookupParameter<IntValue>)Parameters[SamplesEndParameterName]; }
+    }
 
     public ILookupParameter<DoubleValue> NumberOfEvaluatedNodesParameter {
       get { return (ILookupParameter<DoubleValue>)Parameters[NumberOfEvaluatedNodexParameterName]; }
     }
-
+    #endregion
+    #region properties
+    public SymbolicExpressionTree SymbolicExpressionTree {
+      get { return SymbolicExpressionTreeParameter.ActualValue; }
+    }
+    public DataAnalysisProblemData RegressionProblemData {
+      get { return RegressionProblemDataParameter.ActualValue; }
+    }
+    public IntValue SamplesStart {
+      get { return SamplesStartParameter.ActualValue; }
+    }
+    public IntValue SamplesEnd {
+      get { return SamplesEndParameter.ActualValue; }
+    }
     #endregion
 
@@ -75 +90 @@
       Parameters.Add(new LookupParameter<DoubleValue>(QualityParameterName, "The quality of the evaluated symbolic regression solution."));
       Parameters.Add(new LookupParameter<SymbolicExpressionTree>(FunctionTreeParameterName, "The symbolic regression solution encoded as a symbolic expression tree."));
-      Parameters.Add(new LookupParameter<DataAnalysisProblemData>(RegressionProblemDataParameterName, "The data set on which the symbolic regression solution should be evaluated."));
+      Parameters.Add(new LookupParameter<DataAnalysisProblemData>(RegressionProblemDataParameterName, "The problem data on which the symbolic regression solution should be evaluated."));
+      Parameters.Add(new ValueLookupParameter<IntValue>(SamplesStartParameterName, "The start index of the dataset partition on which the symbolic regression solution should be evaluated."));
+      Parameters.Add(new ValueLookupParameter<IntValue>(SamplesEndParameterName, "The end index of the dataset partition on which the symbolic regression solution should be evaluated."));
       Parameters.Add(new LookupParameter<DoubleValue>(NumberOfEvaluatedNodexParameterName, "The number of evaluated nodes so far (for performance measurements.)"));
     }
 
     public override IOperation Apply() {
-      SymbolicExpressionTree solution = FunctionTreeParameter.ActualValue;
-      DataAnalysisProblemData regressionProblemData = RegressionProblemDataParameter.ActualValue;
       DoubleValue numberOfEvaluatedNodes = NumberOfEvaluatedNodesParameter.ActualValue;
-      
-      QualityParameter.ActualValue = new DoubleValue(Evaluate(solution, regressionProblemData.Dataset, regressionProblemData.TargetVariable, regressionProblemData.TrainingSamplesStart, regressionProblemData.TrainingSamplesEnd, numberOfEvaluatedNodes));
+      QualityParameter.ActualValue = new DoubleValue(Evaluate(SymbolicExpressionTree, RegressionProblemData.Dataset,
+        RegressionProblemData.TargetVariable, SamplesStart, SamplesEnd, numberOfEvaluatedNodes));
       return null;
     }

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/SymbolicRegressionProblem.cs

@@ -84 +84 @@
       get { return VisualizerParameter; }
     }
-    public ValueParameter<DoubleValue> BestKnownQualityParameter {
-      get { return (ValueParameter<DoubleValue>)Parameters["BestKnownQuality"]; }
+    public OptionalValueParameter<DoubleValue> BestKnownQualityParameter {
+      get { return (OptionalValueParameter<DoubleValue>)Parameters["BestKnownQuality"]; }
     }
     IParameter ISingleObjectiveProblem.BestKnownQualityParameter {
@@ -147 +147 @@
       Parameters.Add(new ValueParameter<SymbolicExpressionTreeCreator>("SolutionCreator", "The operator which should be used to create new symbolic regression solutions.", creator));
       Parameters.Add(new ValueParameter<ISymbolicRegressionEvaluator>("Evaluator", "The operator which should be used to evaluate symbolic regression solutions.", evaluator));
-      Parameters.Add(new ValueParameter<DoubleValue>("BestKnownQuality", "The minimal error value that can be reached by symbolic regression models.", new DoubleValue(0)));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>("BestKnownQuality", "The minimal error value that reached by symbolic regression solutions for the problem."));
       Parameters.Add(new ValueParameter<ISymbolicExpressionGrammar>("FunctionTreeGrammar", "The grammar that should be used for symbolic regression models.", globalGrammar));
       Parameters.Add(new ValueParameter<IntValue>("MaxExpressionLength", "Maximal length of the symbolic expression.", new IntValue(100)));
@@ -159 +159 @@
       creator.MaxFunctionArgumentsParameter.ActualName = "MaxFunctionArguments";
       creator.MaxFunctionDefinitionsParameter.ActualName = "MaxFunctionDefiningBranches";
-      evaluator.QualityParameter.ActualName = "TrainingMeanSquaredError";
       DataAnalysisProblemDataParameter.ValueChanged += new EventHandler(DataAnalysisProblemDataParameter_ValueChanged);
       DataAnalysisProblemData.ProblemDataChanged += new EventHandler(DataAnalysisProblemData_Changed);
@@ -169 +168 @@
     }
 
-    void DataAnalysisProblemDataParameter_ValueChanged(object sender, EventArgs e) {
-      DataAnalysisProblemData.ProblemDataChanged += new EventHandler(DataAnalysisProblemData_Changed);
-    }
-
-    void DataAnalysisProblemData_Changed(object sender, EventArgs e) {
-      foreach (var varSymbol in FunctionTreeGrammar.Symbols.OfType<HeuristicLab.Problems.DataAnalysis.Symbolic.Symbols.Variable>()) {
-        varSymbol.VariableNames = DataAnalysisProblemData.InputVariables.Select(x => x.Value);
-      }
-    }
 
     [StorableConstructor]
@@ -189 +179 @@
 
     #region Events
+    void DataAnalysisProblemDataParameter_ValueChanged(object sender, EventArgs e) {
+      DataAnalysisProblemData.ProblemDataChanged += new EventHandler(DataAnalysisProblemData_Changed);
+    }
+
+    void DataAnalysisProblemData_Changed(object sender, EventArgs e) {
+      foreach (var varSymbol in FunctionTreeGrammar.Symbols.OfType<HeuristicLab.Problems.DataAnalysis.Symbolic.Symbols.Variable>()) {
+        varSymbol.VariableNames = DataAnalysisProblemData.InputVariables.Select(x => x.Value);
+      }
+      UpdatePartitioningParameters();
+    }
+
+    private void UpdatePartitioningParameters() {
+      int trainingStart = DataAnalysisProblemData.TrainingSamplesStart.Value;
+      int validationEnd = DataAnalysisProblemData.TrainingSamplesEnd.Value;
+      int trainingEnd = trainingStart + (validationEnd - trainingStart) / 2;
+      int validationStart = trainingEnd;
+      var solutionVisualizer = Visualizer as BestValidationSymbolicRegressionSolutionVisualizer;
+      if (solutionVisualizer != null) {
+        solutionVisualizer.ValidationSamplesStartParameter.Value = new IntValue(validationStart);
+        solutionVisualizer.ValidationSamplesEndParameter.Value = new IntValue(validationEnd);
+      }
+      Evaluator.SamplesStartParameter.Value = new IntValue(trainingStart);
+      Evaluator.SamplesEndParameter.Value = new IntValue(trainingEnd);
+    }
+
     public event EventHandler SolutionCreatorChanged;
     private void OnSolutionCreatorChanged() {
@@ -269 +284 @@
     }
     private void ParameterizeEvaluator() {
-      Evaluator.FunctionTreeParameter.ActualName = SolutionCreator.SymbolicExpressionTreeParameter.ActualName;
+      Evaluator.SymbolicExpressionTreeParameter.ActualName = SolutionCreator.SymbolicExpressionTreeParameter.ActualName;
       Evaluator.RegressionProblemDataParameter.ActualName = DataAnalysisProblemDataParameter.Name;
+      Evaluator.QualityParameter.ActualName = "TrainingMeanSquaredError";
+      Evaluator.SamplesStartParameter.Value = new IntValue(DataAnalysisProblemData.TrainingSamplesStart.Value);
+      Evaluator.SamplesEndParameter.Value = new IntValue((DataAnalysisProblemData.TrainingSamplesStart.Value + DataAnalysisProblemData.TrainingSamplesEnd.Value) / 2);
     }
     private void ParameterizeVisualizer() {
@@ -278 +296 @@
           solutionVisualizer.SymbolicExpressionTreeParameter.ActualName = SolutionCreator.SymbolicExpressionTreeParameter.ActualName;
           solutionVisualizer.DataAnalysisProblemDataParameter.ActualName = DataAnalysisProblemDataParameter.Name;
+          solutionVisualizer.ValidationSamplesStartParameter.Value = new IntValue((DataAnalysisProblemData.TrainingSamplesStart.Value + DataAnalysisProblemData.TrainingSamplesEnd.Value) / 2);
+          solutionVisualizer.ValidationSamplesEndParameter.Value = new IntValue(DataAnalysisProblemData.TrainingSamplesEnd.Value);
         }
       }
@@ -289 +309 @@
       }
       foreach (ISymbolicRegressionEvaluator op in Operators.OfType<ISymbolicRegressionEvaluator>()) {
-        op.FunctionTreeParameter.ActualName = SolutionCreator.SymbolicExpressionTreeParameter.ActualName;
+        op.SymbolicExpressionTreeParameter.ActualName = SolutionCreator.SymbolicExpressionTreeParameter.ActualName;
         op.RegressionProblemDataParameter.ActualName = DataAnalysisProblemDataParameter.Name;
         op.NumberOfEvaluatedNodesParameter.ActualName = NumberOfEvaluatedNodesParameter.Name;