source: branches/HeuristicLab.EvolutionTracking/HeuristicLab.EvolutionaryTracking/3.4/Analyzers/BuildingBlocks/RelevantBuildingBlocksAnalyzer.cs @ 10264

using System.Collections.Generic;
using System.Linq;
using HeuristicLab.Analysis;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
using HeuristicLab.Operators;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Problems.DataAnalysis.Symbolic;
using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;

namespace HeuristicLab.EvolutionaryTracking.Analyzers.BuildingBlocks {
  [StorableClass]
  public class RelevantBuildingBlocksAnalyzer : SingleSuccessorOperator, IAnalyzer {
    private const string SymbolicExpressionTreeParameterName = "SymbolicExpressionTree";
    private const string UpdateIntervalParameterName = "UpdateInterval";
    private const string UpdateCounterParameterName = "UpdateCounter";
    private const string ResultsParameterName = "Results";
    private const string MatchVariablesParameterName = "MatchVariableNames";
    private const string MatchVariableWeightsParameterName = "MatchVariableWeights";
    private const string MatchConstantValuesParameterName = "MatchConstantValues";
    private const string SymbolicExpressionTreeGrammarParameterName = "SymbolicExpressionTreeGrammar";
    private const string ClonedSymbolicExpressionTreeGrammarParameterName = "ClonedSymbolicExpressionTreeGrammar";
    private const string SymbolicRegressionSolutionParameterName = "SymbolicRegressionSolution";
    private const string PopulationGraphParameterName = "PopulationGraph";
    private const string FrequentFragmentsTableParameterName = "FrequentFragmentsTable";
    private const string OccurrenceFrequencyTresholdParameterName = "OccurrenceThreshold";

    private readonly Dictionary<string, IFragment> foundFragments;

    #region Parameters
    public ValueParameter<IntValue> UpdateIntervalParameter {
      get { return (ValueParameter<IntValue>)Parameters[UpdateIntervalParameterName]; }
    }

    public ValueParameter<IntValue> UpdateCounterParameter {
      get { return (ValueParameter<IntValue>)Parameters[UpdateCounterParameterName]; }
    }
    public ValueParameter<DoubleValue> OccurrenceThresholdParameter {
      get { return (ValueParameter<DoubleValue>)Parameters[OccurrenceFrequencyTresholdParameterName]; }
    }
    public LookupParameter<ResultCollection> ResultsParameter {
      get { return (LookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
    }
    public IScopeTreeLookupParameter<ISymbolicExpressionTree> SymbolicExpressionTreeParameter {
      get { return (IScopeTreeLookupParameter<ISymbolicExpressionTree>)Parameters[SymbolicExpressionTreeParameterName]; }
    }
    public ValueParameter<BoolValue> MatchVariableNamesParameter {
      get { return (ValueParameter<BoolValue>)Parameters[MatchVariablesParameterName]; }
    }
    public ValueParameter<BoolValue> MatchVariableWeightsParameter {
      get { return (ValueParameter<BoolValue>)Parameters[MatchVariableWeightsParameterName]; }
    }
    public ValueParameter<BoolValue> MatchConstantValuesParameter {
      get { return (ValueParameter<BoolValue>)Parameters[MatchConstantValuesParameterName]; }
    }
    public IValueLookupParameter<ISymbolicExpressionGrammar> SymbolicExpressionTreeGrammarParameter {
      get { return (IValueLookupParameter<ISymbolicExpressionGrammar>)Parameters[SymbolicExpressionTreeGrammarParameterName]; }
    }
    public ILookupParameter<ISymbolicExpressionGrammar> ClonedSymbolicExpressionTreeGrammarParameter {
      get { return (ILookupParameter<ISymbolicExpressionGrammar>)Parameters[ClonedSymbolicExpressionTreeGrammarParameterName]; }
    }
    public ILookupParameter<ISymbolicRegressionSolution> SymbolicRegressionSolutionParameter {
      get { return (ILookupParameter<ISymbolicRegressionSolution>)Parameters[SymbolicRegressionSolutionParameterName]; }
    }
    #endregion

    #region Parameter properties
    public IntValue UpdateInterval {
      get { return UpdateIntervalParameter.Value; }
    }
    public IntValue UpdateCounter {
      get { return UpdateCounterParameter.Value; }
    }
    public DoubleValue OccurrenceThreshold {
      get { return OccurrenceThresholdParameter.Value; }
    }
    public ResultCollection Results {
      get { return ResultsParameter.ActualValue; }
    }
    public ItemArray<ISymbolicExpressionTree> SymbolicExpressionTrees {
      get { return SymbolicExpressionTreeParameter.ActualValue; }
    }
    public ISymbolicExpressionGrammar SymbolicExpressionTreeGrammar {
      get { return ClonedSymbolicExpressionTreeGrammarParameter.ActualValue; }
    }
    public ISymbolicRegressionSolution SymbolicRegressionSolution {
      get { return SymbolicRegressionSolutionParameter.ActualValue; }
    }
    #endregion

    [StorableConstructor]
    protected RelevantBuildingBlocksAnalyzer(bool deserializing) : base(deserializing) { }

    public override IDeepCloneable Clone(Cloner cloner) { return new RelevantBuildingBlocksAnalyzer(this, cloner); }

    protected RelevantBuildingBlocksAnalyzer(RelevantBuildingBlocksAnalyzer original, Cloner cloner) : base(original, cloner) { }

    public RelevantBuildingBlocksAnalyzer() {
      #region Parameters add
      Parameters.Add(new ScopeTreeLookupParameter<ISymbolicExpressionTree>(SymbolicExpressionTreeParameterName, "The symbolic expression trees to analyze."));
      Parameters.Add(new ValueParameter<IntValue>(UpdateIntervalParameterName, "The interval in which the tree length analysis should be applied.", new IntValue(1)));
      Parameters.Add(new ValueParameter<IntValue>(UpdateCounterParameterName, "The value which counts how many times the operator was called since the last update", new IntValue(0)));
      Parameters.Add(new ValueParameter<DoubleValue>(OccurrenceFrequencyTresholdParameterName, "The occurrence threshold for a fragment to be considered a frequenct fragment.", new DoubleValue(0.5)));
      Parameters.Add(new ValueLookupParameter<ResultCollection>(ResultsParameterName, "The results collection where the analysis values should be stored."));
      // matching predicates      
      Parameters.Add(new ValueParameter<BoolValue>(MatchVariablesParameterName, "Specify if the symbolic expression tree comparer should match variable names.", new BoolValue(true)));
      Parameters.Add(new ValueParameter<BoolValue>(MatchVariableWeightsParameterName, "Specify if the symbolic expression tree comparer should match variable weights.", new BoolValue(true)));
      Parameters.Add(new ValueParameter<BoolValue>(MatchConstantValuesParameterName, "Specify if the symbolic expression tree comparer should match constant values.", new BoolValue(true)));
      Parameters.Add(new ValueLookupParameter<ISymbolicExpressionGrammar>(SymbolicExpressionTreeGrammarParameterName, "The tree grammar that defines the correct syntax of symbolic expression trees that should be created."));
      Parameters.Add(new LookupParameter<ISymbolicExpressionGrammar>(ClonedSymbolicExpressionTreeGrammarParameterName, "An immutable clone of the concrete grammar that is actually used to create and manipulate trees."));
      Parameters.Add(new LookupParameter<ISymbolicRegressionSolution>(SymbolicRegressionSolutionParameterName, "The symbolic regression solution."));
      #endregion
      foundFragments = new Dictionary<string, IFragment>();
    }

    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
    }

    public override void ClearState() {
      foundFragments.Clear();
      base.InitializeState();
    }

    public bool EnabledByDefault { get; private set; }

    public override IOperation Apply() {
      UpdateCounter.Value++;
      if (UpdateCounter.Value != UpdateInterval.Value) return base.Apply();
      UpdateCounter.Value = 0;

      //      if (SymbolicRegressionSolution == null) return base.Apply();

      var comparer = new SymbolicExpressionTreeNodeSimilarityComparer {
        MatchConstantValues = MatchConstantValuesParameter.Value.Value,
        MatchVariableNames = MatchVariableNamesParameter.Value.Value,
        MatchVariableWeights = MatchVariableWeightsParameter.Value.Value
      };

      var fragmentComparer = new SymbolicExpressionTreeFragmentSimilarityComparer { SimilarityComparer = comparer };

      //      var bestIndividual = SymbolicRegressionSolution.Model.SymbolicExpressionTree;
      var bestTrainingSolution = (ISymbolicRegressionSolution)Results["Best training solution"].Value;
      var bestIndividual = bestTrainingSolution.Model.SymbolicExpressionTree;
      var subtrees = bestIndividual.IterateNodesPrefix().Where(node => node.SubtreeCount > 0).Select((node, index) => new { Node = node, Index = index });
      var frequentFragments = (from s in subtrees
                               let fragment = new Fragment { Root = s.Node, Index = s.Index }
                               let count = SymbolicExpressionTrees.Count(tree => tree.Root.ContainsFragment(fragment, comparer))
                               let frequency = (double)count / SymbolicExpressionTrees.Length
                               where frequency >= OccurrenceThreshold.Value
                               select fragment).Distinct(fragmentComparer).ToList();

      var graph = (SymbolicExpressionTreeGenealogyGraph)Results[PopulationGraphParameterName].Value;
      DataTable table;

      if (!Results.ContainsKey(FrequentFragmentsTableParameterName)) {
        table = new DataTable(FrequentFragmentsTableParameterName);
        Results.Add(new Result(FrequentFragmentsTableParameterName, table));
      } else {
        table = (DataTable)Results[FrequentFragmentsTableParameterName].Value;
      }

      foreach (var fragment in foundFragments.Values) {
        var prefixString = fragment.Root.ToPrefixString();
        double count = SymbolicExpressionTrees.Count(t => t.Root.ContainsFragment(fragment, comparer));
        double frequency = count / SymbolicExpressionTrees.Length;
        table.Rows[prefixString].Values.Add(frequency);
      }

      var newFragments = frequentFragments.Where(f => !foundFragments.ContainsKey(f.Root.ToPrefixString())).ToList();
      foreach (var fragment in newFragments) {
        var prefixString = fragment.Root.ToPrefixString();
        var row = new DataRow(prefixString);
        row.Values.AddRange(from g in graph.Nodes.Where(n => n.Rank % 1 == 0).GroupBy(n => n.Rank)
                            let count = (double)g.Count(n => n.SymbolicExpressionTree.Root.ContainsFragment(fragment, comparer))
                            let frequency = count / SymbolicExpressionTrees.Length
                            select frequency);
        foundFragments.Add(prefixString, fragment);
        table.Rows.Add(row);
      }

      return base.Apply();
    }
  }
}