source: branches/2988_ModelsOfModels2/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers/ModelsFrequencyAnalyzer.cs @ 16899

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using HEAL.Attic;
using HeuristicLab.Analysis;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using System;
using System.Collections.Generic;
using System.Linq;

namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
  /// <summary>
  /// Calculates the accumulated frequencies of variable-symbols over all trees in the population.
  /// </summary>
  [Item("SymbolicDataAnalysisModelsFrequencyAnalyzer", "Calculates the accumulated frequencies of Model Clusters over all trees in the population.")]
  [StorableType("0A5EAD1D-89E1-4D89-935C-2CBC142834EE")]
  public sealed class ModelsFrequencyAnalyzer : SymbolicDataAnalysisAnalyzer {
    private const string ModelsFrequencyParameterName = "ModelsFrequency";
    private const string AggregateModelParameterName = "AggregateModelClusters";

    #region parameter properties
    [Storable]
    public ILookupParameter<DataTable> ModelFrequencyParameter {
      get { return (ILookupParameter<DataTable>)Parameters[ModelsFrequencyParameterName]; }
    }
    [Storable]
    public IValueLookupParameter<BoolValue> AggregateModelParameter {
      get { return (IValueLookupParameter<BoolValue>)Parameters[AggregateModelParameterName]; }
    }
    #endregion
    #region properties
    public BoolValue AggregateModel {
      get { return AggregateModelParameter.ActualValue; }
      set { AggregateModelParameter.Value = value; }
    }
    public DataTable ModelFrequency {
      get { return ModelFrequencyParameter.ActualValue; }
      set { ModelFrequencyParameter.ActualValue = value; }
    }
    #endregion
    [StorableConstructor]
    private ModelsFrequencyAnalyzer(StorableConstructorFlag _) : base(_) { }
    private ModelsFrequencyAnalyzer(ModelsFrequencyAnalyzer original, Cloner cloner)
      : base(original, cloner) {
    }
    public ModelsFrequencyAnalyzer()
      : base() {
      Parameters.Add(new LookupParameter<DataTable>(ModelsFrequencyParameterName, "The relative Model Clusters reference frequencies aggregated over all trees in the population."));
      Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateModelParameterName, "Switch that determines whether all references to factor Model Clusters should be aggregated regardless of the value. Turn off to analyze all factor variable references with different values separately.", new BoolValue(true)));
    }

    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      // BackwardsCompatibility3.3
      #region Backwards compatible code, remove with 3.4
      if (!Parameters.ContainsKey(AggregateModelParameterName)) {
        Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateModelParameterName, "Switch that determines whether all references to factor Model Clusters should be aggregated regardless of the value. Turn off to analyze all factor Model Clusters references with different values separately.", new BoolValue(true)));
      }
      #endregion
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      return new ModelsFrequencyAnalyzer(this, cloner);
    }

    public override IOperation Apply() {
      ItemArray<ISymbolicExpressionTree> expressions = SymbolicExpressionTreeParameter.ActualValue;
      ResultCollection results = ResultCollection;
      DataTable datatable;
      if (ModelFrequencyParameter.ActualValue == null) {
        datatable = new DataTable("Model frequencies", "Relative frequency of Model references aggregated over the whole population.");
        datatable.VisualProperties.XAxisTitle = "Generation";
        datatable.VisualProperties.YAxisTitle = "Relative Model Frequency";
        ModelFrequencyParameter.ActualValue = datatable;
        results.Add(new Result("Model frequencies", "Relative frequency of Modelreferences aggregated over the whole population.", datatable));
      }
      /* DoubleMatrix map;
       if (results.ContainsKey("My Map")) {
         map = (DoubleMatrix)results["My Map"];
       } else {
         int generations = 100;
         map = new DoubleMatrix(generations, expressions.Length);
       }*/
      datatable = ModelFrequencyParameter.ActualValue;
      // all rows must have the same number of values so we can just take the first
      int numberOfValues = datatable.Rows.Select(r => r.Values.Count).DefaultIfEmpty().First();
      foreach (var pair in CalculateModelFrequency(expressions).OrderByDescending(x => x.Value)) {
        //var pair in CalculateModelFrequency(expressions).OrderByDescending(x => x.Value).Take(10)
        if (!datatable.Rows.ContainsKey(pair.Key)) {
          // initialize a new row for the variable and pad with zeros
          DataRow row = new DataRow(pair.Key, "", Enumerable.Repeat(0.0, numberOfValues));
          row.VisualProperties.StartIndexZero = true;
          datatable.Rows.Add(row);
        }
        datatable.Rows[pair.Key].Values.Add(Math.Round(pair.Value, 3));
      }

      // add a zero for each data row that was not modified in the previous loop 
      foreach (var row in datatable.Rows.Where(r => r.Values.Count != numberOfValues + 1))
        row.Values.Add(0.0);

      return base.Apply();
    }

    public static IEnumerable<KeyValuePair<string, double>> CalculateModelFrequency(IEnumerable<ISymbolicExpressionTree> trees) {
      var modelFrequency = trees
          .SelectMany(t => GetModelReferences(t))
          .GroupBy(pair => pair.Key, pair => pair.Value)
          .ToDictionary(g => g.Key, g => (double)g.Sum());

      double totalNumberOfSymbols = modelFrequency.Values.Sum();

      foreach (var pair in modelFrequency.OrderBy(p => p.Key, new NaturalStringComparer()))
        yield return new KeyValuePair<string, double>(pair.Key, pair.Value / totalNumberOfSymbols);
    }

    private static IEnumerable<KeyValuePair<string, int>> GetModelReferences(ISymbolicExpressionTree tree) {
      Dictionary<string, int> references = new Dictionary<string, int>();
      foreach (var treeNode in tree.IterateNodesPrefix().OfType<TreeModelTreeNode>()) {
        string referenceId = "Model " + treeNode.TreeNumber;
        if (references.ContainsKey(referenceId)) {
          references[referenceId]++;
        } else {
          references[referenceId] = 1;
        }
      }
      return references;
    }
  }
}