source: branches/3057_DynamicALPS/TestProblems/oesr-alps-master/HeuristicLab.Algorithms.OESRALPS/Analyzers/OverfittingSlidingWindowAnalyzer.cs @ 17717

using HEAL.Attic;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.Problems.DataAnalysis;
using HeuristicLab.Problems.DataAnalysis.Symbolic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace HeuristicLab.Algorithms.OESRALPS.Analyzers
{
    [Item("OverfittingSlidingWindowAnalyzer", "An operator that analyzes the correlation of training and validation fitness of symbolic regression models and moves a sliding window if the thresholds are exceeded.")]
    [StorableType("75E112AA-91B2-4BA4-8544-CF2025A65822")]
    public abstract class OverfittingSlidingWindowAnalyzer<T, U> : SlidingWindowAnalyzer<T, U>
        where T : class, ISymbolicDataAnalysisSingleObjectiveEvaluator<U>
        where U : class, IDataAnalysisProblemData
    {
        private const string OverfittingParameterName = "IsOverfitting";
        
        #region parameter properties
        public IScopeTreeLookupParameter<BoolValue> OverfittingParameter {
            get { return (IScopeTreeLookupParameter<BoolValue>)Parameters[OverfittingParameterName]; }
        }
        #endregion

        #region properties
        public ItemArray<BoolValue> Overfitting {
            get { return OverfittingParameter.ActualValue; }
        }
        #endregion

        [StorableConstructor]
        protected OverfittingSlidingWindowAnalyzer(StorableConstructorFlag _) : base(_) { }
        protected OverfittingSlidingWindowAnalyzer(OverfittingSlidingWindowAnalyzer<T, U> original, Cloner cloner) : base(original, cloner) { }
        public OverfittingSlidingWindowAnalyzer()
        {
            Parameters.Add(new ScopeTreeLookupParameter<BoolValue>(OverfittingParameterName, "Boolean indicator for overfitting."));
        }

        public override IOperation Apply()
        {
            if (TrainingPartitionParameter.ActualValue == null
                || IsSystemOverfitting())
                return OnMoveWindow();

            return base.Apply();
        }

        private bool IsSystemOverfitting()
        {
            // TODO think of a suitable strategy
            return Overfitting.Any()
                && IsMajorityOverfitting();
                //&& IsOverfittingMoreThanTwo();
        }

        private bool IsMajorityOverfitting()
        {
            var skipFirstLayerNumb = 2;
            return Overfitting
                .Skip(skipFirstLayerNumb)
                .Where(overfit => overfit.Value)
                .Count() > Math.Ceiling((Overfitting.Count() - skipFirstLayerNumb) / 2.0);
        }

        private bool IsOverfittingMoreThanTwo()
        {
            var skipFirstLayerNumb = 2;
            return Overfitting
                .Skip(skipFirstLayerNumb)
                .Where(overfit => overfit.Value)
                .Count() > 2;
        }
    }
}