using HEAL.Attic; using HeuristicLab.Common; using HeuristicLab.Core; using HeuristicLab.Data; using HeuristicLab.Optimization; using HeuristicLab.Parameters; using HeuristicLab.Problems.DataAnalysis; using HeuristicLab.Problems.DataAnalysis.Symbolic; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; namespace HeuristicLab.Algorithms.OESRALPS.Analyzers { [Item("OverfittingSlidingWindowAnalyzer", "An operator that analyzes the correlation of training and validation fitness of symbolic regression models and moves a sliding window if the thresholds are exceeded.")] [StorableType("75E112AA-91B2-4BA4-8544-CF2025A65822")] public abstract class OverfittingSlidingWindowAnalyzer : SlidingWindowAnalyzer where T : class, ISymbolicDataAnalysisSingleObjectiveEvaluator where U : class, IDataAnalysisProblemData { private const string OverfittingParameterName = "IsOverfitting"; #region parameter properties public IScopeTreeLookupParameter OverfittingParameter { get { return (IScopeTreeLookupParameter)Parameters[OverfittingParameterName]; } } #endregion #region properties public ItemArray Overfitting { get { return OverfittingParameter.ActualValue; } } #endregion [StorableConstructor] protected OverfittingSlidingWindowAnalyzer(StorableConstructorFlag _) : base(_) { } protected OverfittingSlidingWindowAnalyzer(OverfittingSlidingWindowAnalyzer original, Cloner cloner) : base(original, cloner) { } public OverfittingSlidingWindowAnalyzer() { Parameters.Add(new ScopeTreeLookupParameter(OverfittingParameterName, "Boolean indicator for overfitting.")); } public override IOperation Apply() { if (TrainingPartitionParameter.ActualValue == null || IsSystemOverfitting()) return OnMoveWindow(); return base.Apply(); } private bool IsSystemOverfitting() { // TODO think of a suitable strategy return Overfitting.Any() && IsMajorityOverfitting(); //&& IsOverfittingMoreThanTwo(); } private bool IsMajorityOverfitting() { var skipFirstLayerNumb = 2; return Overfitting .Skip(skipFirstLayerNumb) .Where(overfit => overfit.Value) .Count() > Math.Ceiling((Overfitting.Count() - skipFirstLayerNumb) / 2.0); } private bool IsOverfittingMoreThanTwo() { var skipFirstLayerNumb = 2; return Overfitting .Skip(skipFirstLayerNumb) .Where(overfit => overfit.Value) .Count() > 2; } } }