Changeset 5259

Timestamp:

01/08/11 15:36:17 (14 years ago)

Author:

gkronber

Message:

Added calculation of accuracy of best training solution. #1369

File:

• trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/TrainingBestScaledSymbolicRegressionSolutionAnalyzer.cs (modified) (10 diffs)

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/TrainingBestScaledSymbolicRegressionSolutionAnalyzer.cs

@@ -32 +32 @@
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Evaluators;
 
 namespace HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Analyzers {
@@ -44 +45 @@
     private const string MaximizationParameterName = "Maximization";
     private const string CalculateSolutionComplexityParameterName = "CalculateSolutionComplexity";
+    private const string CalculateSolutionAccuracyParameterName = "CalculateSolutionAccuracy";
     private const string SymbolicExpressionTreeInterpreterParameterName = "SymbolicExpressionTreeInterpreter";
     private const string ProblemDataParameterName = "DataAnalysisProblemData";
     private const string UpperEstimationLimitParameterName = "UpperEstimationLimit";
     private const string LowerEstimationLimitParameterName = "LowerEstimationLimit";
-    private const string BestSolutionParameterName = "Best solution (training)";
-    private const string BestSolutionQualityParameterName = "Best solution quality (training)";
-    private const string BestSolutionLengthParameterName = "Best solution length (training)";
-    private const string BestSolutionHeightParameterName = "Best solution height (training)";
-    private const string BestSolutionVariablesParameterName = "Best solution variables (training)"; 
+    private const string BestSolutionParameterName = "Best training solution";
+    private const string BestSolutionQualityParameterName = "Best training solution quality";
+    private const string BestSolutionLengthParameterName = "Best training solution length";
+    private const string BestSolutionHeightParameterName = "Best training solution height";
+    private const string BestSolutionVariablesParameterName = "Best training solution variables";
+    private const string BestSolutionTrainingRSquaredParameterName = "Best training solution R² (training)";
+    private const string BestSolutionTestRSquaredParameterName = "Best training solution R² (test)";
+    private const string BestSolutionTrainingMseParameterName = "Best training solution mean squared error (training)";
+    private const string BestSolutionTestMseParameterName = "Best training solution mean squared error (test)";
+    private const string BestSolutionTrainingRelativeErrorParameterName = "Best training solution relative error (training)";
+    private const string BestSolutionTestRelativeErrorParameterName = "Best training solution relative error (test)";
     private const string ResultsParameterName = "Results";
 
@@ -68 +76 @@
       get { return (IValueParameter<BoolValue>)Parameters[CalculateSolutionComplexityParameterName]; }
     }
+    public IValueParameter<BoolValue> CalculateSolutionAccuracyParameter {
+      get { return (IValueParameter<BoolValue>)Parameters[CalculateSolutionAccuracyParameterName]; }
+    }
     public IValueLookupParameter<ISymbolicExpressionTreeInterpreter> SymbolicExpressionTreeInterpreterParameter {
       get { return (IValueLookupParameter<ISymbolicExpressionTreeInterpreter>)Parameters[SymbolicExpressionTreeInterpreterParameterName]; }
@@ -95 +106 @@
     public ILookupParameter<IntValue> BestSolutionVariablesParameter {
       get { return (ILookupParameter<IntValue>)Parameters[BestSolutionVariablesParameterName]; }
-    }    
+    }
+    public ILookupParameter<DoubleValue> BestSolutionTrainingRSquaredParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionTrainingRSquaredParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> BestSolutionTestRSquaredParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionTestRSquaredParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> BestSolutionTrainingMseParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionTrainingMseParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> BestSolutionTestMseParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionTestMseParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> BestSolutionTrainingRelativeErrorParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionTrainingRelativeErrorParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> BestSolutionTestRelativeErrorParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[BestSolutionTestRelativeErrorParameterName]; }
+    }
     public ILookupParameter<ResultCollection> ResultsParameter {
       get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
@@ -114 +143 @@
       set { CalculateSolutionComplexityParameter.Value = value; }
     }
+    public BoolValue CalculateSolutionAccuracy {
+      get { return CalculateSolutionAccuracyParameter.Value; }
+      set { CalculateSolutionAccuracyParameter.Value = value; }
+    }
     public ISymbolicExpressionTreeInterpreter SymbolicExpressionTreeInterpreter {
       get { return SymbolicExpressionTreeInterpreterParameter.ActualValue; }
@@ -149 +182 @@
       set { BestSolutionVariablesParameter.ActualValue = value; }
     }
-
+    public DoubleValue BestSolutionTrainingRSquared {
+      get { return BestSolutionTrainingRSquaredParameter.ActualValue; }
+      set { BestSolutionTrainingRSquaredParameter.ActualValue = value; }
+    }
+    public DoubleValue BestSolutionTestRSquared {
+      get { return BestSolutionTestRSquaredParameter.ActualValue; }
+      set { BestSolutionTestRSquaredParameter.ActualValue = value; }
+    }
+    public DoubleValue BestSolutionTrainingMse {
+      get { return BestSolutionTrainingMseParameter.ActualValue; }
+      set { BestSolutionTrainingMseParameter.ActualValue = value; }
+    }
+    public DoubleValue BestSolutionTestMse {
+      get { return BestSolutionTestMseParameter.ActualValue; }
+      set { BestSolutionTestMseParameter.ActualValue = value; }
+    }
+    public DoubleValue BestSolutionTrainingRelativeError {
+      get { return BestSolutionTrainingRelativeErrorParameter.ActualValue; }
+      set { BestSolutionTrainingRelativeErrorParameter.ActualValue = value; }
+    }
+    public DoubleValue BestSolutionTestRelativeError {
+      get { return BestSolutionTestRelativeErrorParameter.ActualValue; }
+      set { BestSolutionTestRelativeErrorParameter.ActualValue = value; }
+    }
     #endregion
 
@@ -161 +217 @@
       Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>(QualityParameterName, "The qualities of the symbolic expression trees to analyze."));
       Parameters.Add(new ValueParameter<BoolValue>(CalculateSolutionComplexityParameterName, "Determines if the length and height of the training best solution should be calculated.", new BoolValue(false)));
+      Parameters.Add(new ValueParameter<BoolValue>(CalculateSolutionAccuracyParameterName, "Determines if the accuracy of the training best solution on the training and test set should be calculated.", new BoolValue(false)));
       Parameters.Add(new ValueLookupParameter<ISymbolicExpressionTreeInterpreter>(SymbolicExpressionTreeInterpreterParameterName, "The interpreter that should be used for the analysis of symbolic expression trees."));
       Parameters.Add(new ValueLookupParameter<DataAnalysisProblemData>(ProblemDataParameterName, "The problem data for which the symbolic expression tree is a solution."));
@@ -170 +227 @@
       Parameters.Add(new LookupParameter<IntValue>(BestSolutionHeightParameterName, "The height of the best symbolic regression solution."));
       Parameters.Add(new LookupParameter<IntValue>(BestSolutionVariablesParameterName, "The number of variables used by the best symbolic regression solution."));
+      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionTrainingRSquaredParameterName, "The R² value on the training set of the best symbolic regression solution."));
+      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionTestRSquaredParameterName, "The R² value on the test set of the best symbolic regression solution."));
+      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionTrainingMseParameterName, "The mean squared error on the training set of the best symbolic regression solution."));
+      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionTestMseParameterName, "The mean squared error value on the test set of the best symbolic regression solution."));
+      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionTrainingRelativeErrorParameterName, "The relative error on the training set of the best symbolic regression solution."));
+      Parameters.Add(new LookupParameter<DoubleValue>(BestSolutionTestRelativeErrorParameterName, "The relative error value on the test set of the best symbolic regression solution."));
       Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection where the best symbolic regression solution should be stored."));
     }
@@ -235 +298 @@
             Results[BestSolutionLengthParameterName].Value = BestSolutionLength;
             Results[BestSolutionHeightParameterName].Value = BestSolutionHeight;
-            Results[BestSolutionVariablesParameterName].Value = BestSolutionHeight;
+            Results[BestSolutionVariablesParameterName].Value = BestSolutionVariables;
           }
+        }
+
+        if (CalculateSolutionAccuracy.Value) {
+          #region update R2,MSE, Rel Error
+          IEnumerable<double> trainingValues = ProblemData.Dataset.GetEnumeratedVariableValues(ProblemData.TargetVariable.Value, ProblemData.TrainingIndizes);
+          IEnumerable<double> testValues = ProblemData.Dataset.GetEnumeratedVariableValues(ProblemData.TargetVariable.Value, ProblemData.TestIndizes);
+          OnlineMeanSquaredErrorEvaluator mseEvaluator = new OnlineMeanSquaredErrorEvaluator();
+          OnlineMeanAbsolutePercentageErrorEvaluator relErrorEvaluator = new OnlineMeanAbsolutePercentageErrorEvaluator();
+          OnlinePearsonsRSquaredEvaluator r2Evaluator = new OnlinePearsonsRSquaredEvaluator();
+
+          #region training
+          var originalEnumerator = trainingValues.GetEnumerator();
+          var estimatedEnumerator = solution.EstimatedTrainingValues.GetEnumerator();
+          while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
+            mseEvaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
+            r2Evaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
+            relErrorEvaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
+          }
+          double trainingR2 = r2Evaluator.RSquared;
+          double trainingMse = mseEvaluator.MeanSquaredError;
+          double trainingRelError = relErrorEvaluator.MeanAbsolutePercentageError;
+          #endregion
+
+          mseEvaluator.Reset();
+          relErrorEvaluator.Reset();
+          r2Evaluator.Reset();
+
+          #region test
+          originalEnumerator = testValues.GetEnumerator();
+          estimatedEnumerator = solution.EstimatedTestValues.GetEnumerator();
+          while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
+            mseEvaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
+            r2Evaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
+            relErrorEvaluator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
+          }
+          double testR2 = r2Evaluator.RSquared;
+          double testMse = mseEvaluator.MeanSquaredError;
+          double testRelError = relErrorEvaluator.MeanAbsolutePercentageError;
+          #endregion
+          BestSolutionTrainingRSquared = new DoubleValue(trainingR2);
+          BestSolutionTestRSquared = new DoubleValue(testR2);
+          BestSolutionTrainingMse = new DoubleValue(trainingMse);
+          BestSolutionTestMse = new DoubleValue(testMse);
+          BestSolutionTrainingRelativeError = new DoubleValue(trainingRelError);
+          BestSolutionTestRelativeError = new DoubleValue(testRelError);
+
+          if (!Results.ContainsKey(BestSolutionTrainingRSquaredParameterName)) {
+            Results.Add(new Result(BestSolutionTrainingRSquaredParameterName, BestSolutionTrainingRSquared));
+            Results.Add(new Result(BestSolutionTestRSquaredParameterName, BestSolutionTestRSquared));
+            Results.Add(new Result(BestSolutionTrainingMseParameterName, BestSolutionTrainingMse));
+            Results.Add(new Result(BestSolutionTestMseParameterName, BestSolutionTestMse));
+            Results.Add(new Result(BestSolutionTrainingRelativeErrorParameterName, BestSolutionTrainingRelativeError));
+            Results.Add(new Result(BestSolutionTestRelativeErrorParameterName, BestSolutionTestRelativeError));
+          } else {
+            Results[BestSolutionTrainingRSquaredParameterName].Value = BestSolutionTrainingRSquared;
+            Results[BestSolutionTestRSquaredParameterName].Value = BestSolutionTestRSquared;
+            Results[BestSolutionTrainingMseParameterName].Value = BestSolutionTrainingMse;
+            Results[BestSolutionTestMseParameterName].Value = BestSolutionTestMse;
+            Results[BestSolutionTrainingRelativeErrorParameterName].Value = BestSolutionTrainingRelativeError;
+            Results[BestSolutionTestRelativeErrorParameterName].Value = BestSolutionTestRelativeError;
+          }
+          #endregion
         }
 
@@ -244 +369 @@
         } else {
           Results[BestSolutionQualityParameterName].Value = BestSolutionQuality;
+          Results[BestSolutionParameterName].Value = BestSolution;
         }
       }