source: branches/2988_ModelsOfModels2/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers/TerminalNodesFrequencyAnalyzer.cs @ 16899

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using HEAL.Attic;
using HeuristicLab.Analysis;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using System;
using System.Collections.Generic;
using System.Linq;

namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
  /// <summary>
  /// Calculates the accumulated frequencies of variable-symbols over all trees in the population.
  /// </summary>
  [Item("TerminalNodesFrequencyAnalyzer", "Calculates the accumulated frequencies of Terminal Nodes over all trees in the population.")]
  [StorableType("638ECFFA-B441-4099-AB5F-DFCA1FE41154")]
  public sealed class TerminalNodesFrequencyAnalyzer : SymbolicDataAnalysisAnalyzer {
    private const string TerminalNodesFrequencyParameterName = "TerminalNodesFrequency";
    private const string AggregateTerminalNodesParameterName = "AggregateTerminalNodes";

    #region parameter properties
    [Storable]
    public ILookupParameter<DataTable> TerminalNodesFrequencyParameter {
      get { return (ILookupParameter<DataTable>)Parameters[TerminalNodesFrequencyParameterName]; }
    }
    [Storable]
    public IValueLookupParameter<BoolValue> AggregateTerminalNodesParameter {
      get { return (IValueLookupParameter<BoolValue>)Parameters[AggregateTerminalNodesParameterName]; }
    }
    #endregion
    #region properties
    public BoolValue AggregateTerminalNodes {
      get { return AggregateTerminalNodesParameter.ActualValue; }
      set { AggregateTerminalNodesParameter.Value = value; }
    }
    public DataTable TerminalNodesFrequency {
      get { return TerminalNodesFrequencyParameter.ActualValue; }
      set { TerminalNodesFrequencyParameter.ActualValue = value; }
    }

    #endregion
    [StorableConstructor]
    private TerminalNodesFrequencyAnalyzer(StorableConstructorFlag _) : base(_) { }
    private TerminalNodesFrequencyAnalyzer(TerminalNodesFrequencyAnalyzer original, Cloner cloner)
      : base(original, cloner) {
    }
    public TerminalNodesFrequencyAnalyzer()
      : base() {
      Parameters.Add(new LookupParameter<DataTable>(TerminalNodesFrequencyParameterName, "The relative Terminal Nodes reference frequencies aggregated over all trees in the population."));
      Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateTerminalNodesParameterName, "Switch that determines whether all references to factor Terminal Nodes should be aggregated regardless of the value. Turn off to analyze all factor variable references with different values separately.", new BoolValue(true)));
    }

    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      // BackwardsCompatibility3.3
      #region Backwards compatible code, remove with 3.4
      if (!Parameters.ContainsKey(AggregateTerminalNodesParameterName)) {
        Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateTerminalNodesParameterName, "Switch that determines whether all references to factor Terminal Nodes should be aggregated regardless of the value. Turn off to analyze all factor Terminal Nodes references with different values separately.", new BoolValue(true)));
      }
      #endregion
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      return new TerminalNodesFrequencyAnalyzer(this, cloner);
    }

    public override IOperation Apply() {
      ItemArray<ISymbolicExpressionTree> expressions = SymbolicExpressionTreeParameter.ActualValue;
      ResultCollection results = ResultCollection;
      DataTable datatable;
      if (TerminalNodesFrequencyParameter.ActualValue == null) {
        datatable = new DataTable("Terminal Nodes frequencies", "Relative frequency of Terminal Nodes references aggregated over the whole population.");
        datatable.VisualProperties.XAxisTitle = "Generation";
        datatable.VisualProperties.YAxisTitle = "Relative Model Frequency";
        TerminalNodesFrequencyParameter.ActualValue = datatable;
        results.Add(new Result("Terminal Nodes frequencies", "Relative frequency of Terminal Nodes references aggregated over the whole population.", datatable));
      }

      datatable = TerminalNodesFrequencyParameter.ActualValue;
      // all rows must have the same number of values so we can just take the first
      int numberOfValues = datatable.Rows.Select(r => r.Values.Count).DefaultIfEmpty().First();
      foreach (var pair in CalculateTerminalNodesFrequency(expressions).OrderByDescending(x => x.Value)) {
        //var pair in CalculateModelFrequency(expressions).OrderByDescending(x => x.Value).Take(10)
        if (!datatable.Rows.ContainsKey(pair.Key)) {
          // initialize a new row for the variable and pad with zeros
          DataRow row = new DataRow(pair.Key, "", Enumerable.Repeat(0.0, numberOfValues));
          row.VisualProperties.StartIndexZero = true;
          datatable.Rows.Add(row);
        }
        datatable.Rows[pair.Key].Values.Add(Math.Round(pair.Value, 3));
      }

      // add a zero for each data row that was not modified in the previous loop 
      foreach (var row in datatable.Rows.Where(r => r.Values.Count != numberOfValues + 1))
        row.Values.Add(0.0);

      return base.Apply();
    }

    public static IEnumerable<KeyValuePair<string, double>> CalculateTerminalNodesFrequency(IEnumerable<ISymbolicExpressionTree> trees) {
      var terminalNodesFrequency = trees
          .SelectMany(t => GetTerminalNodesReferences(t))
          .GroupBy(pair => pair.Key, pair => pair.Value)
          .ToDictionary(g => g.Key, g => (double)g.Sum());

      double totalNumberOfSymbols = terminalNodesFrequency.Values.Sum();

      foreach (var pair in terminalNodesFrequency.OrderBy(p => p.Key, new NaturalStringComparer()))
        yield return new KeyValuePair<string, double>(pair.Key, pair.Value / totalNumberOfSymbols);
    }

    private static IEnumerable<KeyValuePair<string, int>> GetTerminalNodesReferences(ISymbolicExpressionTree tree) {
      Dictionary<string, int> references = new Dictionary<string, int>();
      foreach (var treeNode in tree.IterateNodesPrefix().OfType<VariableTreeNode>()) {
        string referenceId = "Variable ";
        if (references.ContainsKey(referenceId)) {
          references[referenceId]++;
        } else {
          references[referenceId] = 1;
        }
      }
      foreach (var treeNode in tree.IterateNodesPrefix().OfType<TreeModelTreeNode>()) {
        string referenceId = "Model ";
        if (references.ContainsKey(referenceId)) {
          references[referenceId]++;
        } else {
          references[referenceId] = 1;
        }
      }
      foreach (var treeNode in tree.IterateNodesPrefix().OfType<ConstantTreeNode>()) {
        string referenceId = "Constant";
        if (references.ContainsKey(referenceId)) {
          references[referenceId]++;
        } else {
          references[referenceId] = 1;
        }
      }
      return references;
    }
  }
}