Changeset 5747 for branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective

Timestamp:

03/18/11 10:01:00 (13 years ago)

Author:

gkronber

Message:

#1418 renamed bounded evaluator, added base classes for single objective and multi objective validation analzers, added overfitting analyzers for symbolic regression and classification.

Location:

branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective

Files:

• SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator.cs (moved) (moved from branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectiveBoundeMeanSquaredErrorEvaluator.cs) (1 diff)
• SymbolicClassificationSingleObjectiveMeanSquaredErrorEvaluator.cs (modified) (2 diffs)
• SymbolicClassificationSingleObjectiveOverfittingAnalyzer.cs (copied) (copied from branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/Analyzers/SymbolicRegressionOverfittingAnalyzer.cs) (5 diffs)
• SymbolicClassificationSingleObjectivePearsonRSquaredEvaluator.cs (modified) (2 diffs)
• SymbolicClassificationSingleObjectiveProblem.cs (modified) (1 diff)

branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator.cs

@@ -32 +32 @@
   [Item("Bounded Mean squared error Evaluator", "Calculates the bounded mean squared error of a symbolic classification solution (estimations above or below the class values are only penaltilized linearly.")]
   [StorableClass]
-  public class SymbolicClassificationSingleObjectiveBoundeMeanSquaredErrorEvaluator : SymbolicClassificationSingleObjectiveEvaluator {
+  public class SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator : SymbolicClassificationSingleObjectiveEvaluator {
 
     [StorableConstructor]
-    protected SymbolicClassificationSingleObjectiveBoundeMeanSquaredErrorEvaluator(bool deserializing) : base(deserializing) { }
-    protected SymbolicClassificationSingleObjectiveBoundeMeanSquaredErrorEvaluator(SymbolicClassificationSingleObjectiveBoundeMeanSquaredErrorEvaluator original, Cloner cloner) : base(original, cloner) { }
+    protected SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator(bool deserializing) : base(deserializing) { }
+    protected SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator(SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator original, Cloner cloner) : base(original, cloner) { }
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new SymbolicClassificationSingleObjectiveBoundeMeanSquaredErrorEvaluator(this, cloner);
+      return new SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator(this, cloner);
     }
 
-    public SymbolicClassificationSingleObjectiveBoundeMeanSquaredErrorEvaluator() : base() { }
+    public SymbolicClassificationSingleObjectiveBoundedMeanSquaredErrorEvaluator() : base() { }
 
     public override bool Maximization { get { return false; } }

branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectiveMeanSquaredErrorEvaluator.cs

@@ -26 +26 @@
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using System;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Classification {
@@ -62 +63 @@
       LowerEstimationLimitParameter.ExecutionContext = context;
       UpperEstimationLimitParameter.ExecutionContext = context;
-      
+
       double mse = Calculate(SymbolicDataAnalysisTreeInterpreter, tree, LowerEstimationLimit.Value, UpperEstimationLimit.Value, problemData, rows);
 

branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectiveOverfittingAnalyzer.cs

@@ -31 +31 @@
 using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Problems.DataAnalysis.Evaluators;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
 using System;
 
-namespace HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Analyzers {
-  [Item("SymbolicRegressionOverfittingAnalyzer", "Calculates and tracks correlation of training and validation fitness of symbolic regression models.")]
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Classification {
+  [Item("SymbolicClassificationSingleObjectiveOverfittingAnalyzer", "Calculates and tracks correlation of training and validation fitness of symbolic classification models.")]
   [StorableClass]
-  public sealed class SymbolicRegressionOverfittingAnalyzer : SymbolicRegressionValidationAnalyzer, ISymbolicRegressionAnalyzer {
-    private const string MaximizationParameterName = "Maximization";
-    private const string QualityParameterName = "Quality";
-    private const string TrainingValidationCorrelationParameterName = "TrainingValidationCorrelation";
-    private const string TrainingValidationCorrelationTableParameterName = "TrainingValidationCorrelationTable";
+  public sealed class SymbolicClassificationSingleObjectiveOverfittingAnalyzer : SymbolicDataAnalysisSingleObjectiveValidationAnalyzer<ISymbolicClassificationSingleObjectiveEvaluator, IClassificationProblemData> {
+    private const string TrainingValidationCorrelationParameterName = "Training and validation fitness correlation";
+    private const string TrainingValidationCorrelationTableParameterName = "Training and validation fitness correlation table";
     private const string LowerCorrelationThresholdParameterName = "LowerCorrelationThreshold";
     private const string UpperCorrelationThresholdParameterName = "UpperCorrelationThreshold";
     private const string OverfittingParameterName = "IsOverfitting";
-    private const string ResultsParameterName = "Results";
 
     #region parameter properties
-    public ScopeTreeLookupParameter<DoubleValue> QualityParameter {
-      get { return (ScopeTreeLookupParameter<DoubleValue>)Parameters[QualityParameterName]; }
-    }
-    public ILookupParameter<BoolValue> MaximizationParameter {
-      get { return (ILookupParameter<BoolValue>)Parameters[MaximizationParameterName]; }
-    }
     public ILookupParameter<DoubleValue> TrainingValidationQualityCorrelationParameter {
       get { return (ILookupParameter<DoubleValue>)Parameters[TrainingValidationCorrelationParameterName]; }
@@ -70 +60 @@
       get { return (ILookupParameter<BoolValue>)Parameters[OverfittingParameterName]; }
     }
-    public ILookupParameter<ResultCollection> ResultsParameter {
-      get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
-    }
-    #endregion
-    #region properties
-    public BoolValue Maximization {
-      get { return MaximizationParameter.ActualValue; }
-    }
     #endregion
 
     [StorableConstructor]
-    private SymbolicRegressionOverfittingAnalyzer(bool deserializing) : base(deserializing) { }
-    private SymbolicRegressionOverfittingAnalyzer(SymbolicRegressionOverfittingAnalyzer original, Cloner cloner) : base(original, cloner) { }
-    public SymbolicRegressionOverfittingAnalyzer()
+    private SymbolicClassificationSingleObjectiveOverfittingAnalyzer(bool deserializing) : base(deserializing) { }
+    private SymbolicClassificationSingleObjectiveOverfittingAnalyzer(SymbolicClassificationSingleObjectiveOverfittingAnalyzer original, Cloner cloner) : base(original, cloner) { }
+    public SymbolicClassificationSingleObjectiveOverfittingAnalyzer()
       : base() {
-      Parameters.Add(new ScopeTreeLookupParameter<DoubleValue>(QualityParameterName, "Training fitness"));
-      Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName, "The direction of optimization."));
       Parameters.Add(new LookupParameter<DoubleValue>(TrainingValidationCorrelationParameterName, "Correlation of training and validation fitnesses"));
       Parameters.Add(new LookupParameter<DataTable>(TrainingValidationCorrelationTableParameterName, "Data table of training and validation fitness correlation values over the whole run."));
@@ -92 +72 @@
       Parameters.Add(new ValueLookupParameter<DoubleValue>(UpperCorrelationThresholdParameterName, "Upper threshold for correlation value that marks the boundary from overfitting to non-overfitting.", new DoubleValue(0.75)));
       Parameters.Add(new LookupParameter<BoolValue>(OverfittingParameterName, "Boolean indicator for overfitting."));
-      Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The results collection."));
-    }
-
-    [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() {
     }
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new SymbolicRegressionOverfittingAnalyzer(this, cloner);
+      return new SymbolicClassificationSingleObjectiveOverfittingAnalyzer(this, cloner);
     }
 
-    protected override void Analyze(SymbolicExpressionTree[] trees, double[] validationQuality) {
+    public override IOperation Apply() {
       double[] trainingQuality = QualityParameter.ActualValue.Select(x => x.Value).ToArray();
-
+      // evaluate on validation partition
+      int start = ValidationSamplesStart.Value;
+      int end = ValidationSamplesEnd.Value;
+      var rows = Enumerable.Range(start, end - start);
+      IExecutionContext childContext = (IExecutionContext)ExecutionContext.CreateChildOperation(Evaluator);
+      double[] validationQuality = (from tree in SymbolicExpressionTrees
+                                    select Evaluator.Evaluate(childContext, tree, ProblemData, rows))
+                                   .ToArray();
       double r = alglib.spearmancorr2(trainingQuality, validationQuality);
 
@@ -111 +93 @@
 
       if (TrainingValidationQualityCorrelationTableParameter.ActualValue == null) {
-        var dataTable = new DataTable("Training and validation fitness correlation table", "Data table of training and validation fitness correlation values over the whole run.");
-        dataTable.Rows.Add(new DataRow("Training and validation fitness correlation", "Training and validation fitness correlation values"));
+        var dataTable = new DataTable(TrainingValidationQualityCorrelationTableParameter.Name, TrainingValidationQualityCorrelationTableParameter.Description);
+        dataTable.Rows.Add(new DataRow(TrainingValidationQualityCorrelationParameter.Name, TrainingValidationQualityCorrelationParameter.Description));
         TrainingValidationQualityCorrelationTableParameter.ActualValue = dataTable;
-        ResultsParameter.ActualValue.Add(new Result(TrainingValidationCorrelationTableParameterName, dataTable));
+        ResultCollectionParameter.ActualValue.Add(new Result(TrainingValidationQualityCorrelationTableParameter.Name, dataTable));
       }
 
-      TrainingValidationQualityCorrelationTableParameter.ActualValue.Rows["Training and validation fitness correlation"].Values.Add(r);
+      TrainingValidationQualityCorrelationTableParameter.ActualValue.Rows[TrainingValidationQualityCorrelationParameter.Name].Values.Add(r);
 
       if (OverfittingParameter.ActualValue != null && OverfittingParameter.ActualValue.Value) {
@@ -128 +110 @@
         OverfittingParameter.ActualValue = new BoolValue(r < LowerCorrelationThresholdParameter.ActualValue.Value);
       }
+
+      return base.Apply();
     }
   }

branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectivePearsonRSquaredEvaluator.cs

@@ -26 +26 @@
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using System;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Classification {
@@ -54 +55 @@
       IEnumerable<double> estimatedValues = interpreter.GetSymbolicExpressionTreeValues(solution, problemData.Dataset, rows);
       IEnumerable<double> originalValues = problemData.Dataset.GetEnumeratedVariableValues(problemData.TargetVariable, rows);
-      return OnlinePearsonsRSquaredEvaluator.Calculate(originalValues, estimatedValues);
+      try {
+        return OnlinePearsonsRSquaredEvaluator.Calculate(originalValues, estimatedValues);
+      }
+      catch (ArgumentException) {
+        // if R² cannot be calculated because of NaN or ininity elements => return worst possible fitness valuse
+        return 0.0;
+      }
     }
 

branches/DataAnalysis Refactoring/HeuristicLab.Problems.DataAnalysis.Symbolic.Classification/3.4/SingleObjective/SymbolicClassificationSingleObjectiveProblem.cs

@@ -76 +76 @@
       Operators.Add(new SymbolicClassificationSingleObjectiveTrainingBestSolutionAnalyzer());
       Operators.Add(new SymbolicClassificationSingleObjectiveValidationBestSolutionAnalyzer());
+      Operators.Add(new SymbolicClassificationSingleObjectiveOverfittingAnalyzer());
       ParameterizeOperators();
     }