#region License Information /* HeuristicLab * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL) * * This file is part of HeuristicLab. * * HeuristicLab is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * HeuristicLab is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with HeuristicLab. If not, see . */ #endregion using System.Collections.Generic; using System.Linq; using HeuristicLab.Analysis; using HeuristicLab.Common; using HeuristicLab.Core; using HeuristicLab.Data; using HeuristicLab.Optimization; using HeuristicLab.Parameters; using HEAL.Attic; namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression { [Item("SymbolicRegressionSingleObjectiveOverfittingAnalyzer", "Calculates and tracks correlation of training and validation fitness of symbolic regression models.")] [StorableType("AE1F0A23-BEB1-47AF-8ECF-DBCFD32AA5A9")] public sealed class SymbolicRegressionSingleObjectiveOverfittingAnalyzer : SymbolicDataAnalysisSingleObjectiveValidationAnalyzer { private const string TrainingValidationCorrelationParameterName = "Training and validation fitness correlation"; private const string TrainingValidationCorrelationTableParameterName = "Training and validation fitness correlation table"; private const string LowerCorrelationThresholdParameterName = "LowerCorrelationThreshold"; private const string UpperCorrelationThresholdParameterName = "UpperCorrelationThreshold"; private const string OverfittingParameterName = "IsOverfitting"; #region parameter properties public ILookupParameter TrainingValidationQualityCorrelationParameter { get { return (ILookupParameter)Parameters[TrainingValidationCorrelationParameterName]; } } public ILookupParameter TrainingValidationQualityCorrelationTableParameter { get { return (ILookupParameter)Parameters[TrainingValidationCorrelationTableParameterName]; } } public IValueLookupParameter LowerCorrelationThresholdParameter { get { return (IValueLookupParameter)Parameters[LowerCorrelationThresholdParameterName]; } } public IValueLookupParameter UpperCorrelationThresholdParameter { get { return (IValueLookupParameter)Parameters[UpperCorrelationThresholdParameterName]; } } public ILookupParameter OverfittingParameter { get { return (ILookupParameter)Parameters[OverfittingParameterName]; } } #endregion [StorableConstructor] private SymbolicRegressionSingleObjectiveOverfittingAnalyzer(StorableConstructorFlag _) : base(_) { } private SymbolicRegressionSingleObjectiveOverfittingAnalyzer(SymbolicRegressionSingleObjectiveOverfittingAnalyzer original, Cloner cloner) : base(original, cloner) { } public SymbolicRegressionSingleObjectiveOverfittingAnalyzer() : base() { Parameters.Add(new LookupParameter(TrainingValidationCorrelationParameterName, "Correlation of training and validation fitnesses")); Parameters.Add(new LookupParameter(TrainingValidationCorrelationTableParameterName, "Data table of training and validation fitness correlation values over the whole run.")); Parameters.Add(new ValueLookupParameter(LowerCorrelationThresholdParameterName, "Lower threshold for correlation value that marks the boundary from non-overfitting to overfitting.", new DoubleValue(0.65))); Parameters.Add(new ValueLookupParameter(UpperCorrelationThresholdParameterName, "Upper threshold for correlation value that marks the boundary from overfitting to non-overfitting.", new DoubleValue(0.75))); Parameters.Add(new LookupParameter(OverfittingParameterName, "Boolean indicator for overfitting.")); } public override IDeepCloneable Clone(Cloner cloner) { return new SymbolicRegressionSingleObjectiveOverfittingAnalyzer(this, cloner); } public override IOperation Apply() { IEnumerable rows = GenerateRowsToEvaluate(); if (!rows.Any()) return base.Apply(); double[] trainingQuality = QualityParameter.ActualValue.Select(x => x.Value).ToArray(); var problemData = ProblemDataParameter.ActualValue; var evaluator = EvaluatorParameter.ActualValue; // evaluate on validation partition IExecutionContext childContext = (IExecutionContext)ExecutionContext.CreateChildOperation(evaluator); double[] validationQuality = SymbolicExpressionTree .Select(t => evaluator.Evaluate(childContext, t, problemData, rows)) .ToArray(); double r = 0.0; try { r = alglib.spearmancorr2(trainingQuality, validationQuality); } catch (alglib.alglibexception) { r = 0.0; } TrainingValidationQualityCorrelationParameter.ActualValue = new DoubleValue(r); if (TrainingValidationQualityCorrelationTableParameter.ActualValue == null) { var dataTable = new DataTable(TrainingValidationQualityCorrelationTableParameter.Name, TrainingValidationQualityCorrelationTableParameter.Description); dataTable.Rows.Add(new DataRow(TrainingValidationQualityCorrelationParameter.Name, TrainingValidationQualityCorrelationParameter.Description)); dataTable.Rows[TrainingValidationQualityCorrelationParameter.Name].VisualProperties.StartIndexZero = true; TrainingValidationQualityCorrelationTableParameter.ActualValue = dataTable; ResultCollectionParameter.ActualValue.Add(new Result(TrainingValidationQualityCorrelationTableParameter.Name, dataTable)); } TrainingValidationQualityCorrelationTableParameter.ActualValue.Rows[TrainingValidationQualityCorrelationParameter.Name].Values.Add(r); if (OverfittingParameter.ActualValue != null && OverfittingParameter.ActualValue.Value) { // overfitting == true // => r must reach the upper threshold to switch back to non-overfitting state OverfittingParameter.ActualValue = new BoolValue(r < UpperCorrelationThresholdParameter.ActualValue.Value); } else { // overfitting == false // => r must drop below lower threshold to switch to overfitting state OverfittingParameter.ActualValue = new BoolValue(r < LowerCorrelationThresholdParameter.ActualValue.Value); } return base.Apply(); } } }