1 | #region License Information
|
---|
2 | /* HeuristicLab
|
---|
3 | * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
4 | *
|
---|
5 | * This file is part of HeuristicLab.
|
---|
6 | *
|
---|
7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
8 | * it under the terms of the GNU General Public License as published by
|
---|
9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
10 | * (at your option) any later version.
|
---|
11 | *
|
---|
12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
15 | * GNU General Public License for more details.
|
---|
16 | *
|
---|
17 | * You should have received a copy of the GNU General Public License
|
---|
18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
19 | */
|
---|
20 | #endregion
|
---|
21 |
|
---|
22 | using System;
|
---|
23 | using System.Collections.Generic;
|
---|
24 | using System.Linq;
|
---|
25 | using HeuristicLab.Analysis;
|
---|
26 | using HeuristicLab.Common;
|
---|
27 | using HeuristicLab.Core;
|
---|
28 | using HeuristicLab.Data;
|
---|
29 | using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
|
---|
30 | using HeuristicLab.Optimization;
|
---|
31 | using HeuristicLab.Parameters;
|
---|
32 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
33 |
|
---|
34 | namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
|
---|
35 | /// <summary>
|
---|
36 | /// Calculates the accumulated frequencies of variable-symbols over all trees in the population.
|
---|
37 | /// </summary>
|
---|
38 | [Item("SymbolicDataAnalysisVariableFrequencyAnalyzer", "Calculates the accumulated frequencies of variable-symbols over all trees in the population.")]
|
---|
39 | [StorableClass]
|
---|
40 | public sealed class SymbolicDataAnalysisVariableFrequencyAnalyzer : SymbolicDataAnalysisAnalyzer {
|
---|
41 | private const string VariableFrequenciesParameterName = "VariableFrequencies";
|
---|
42 | private const string AggregateLaggedVariablesParameterName = "AggregateLaggedVariables";
|
---|
43 | private const string VariableImpactsParameterName = "VariableImpacts";
|
---|
44 |
|
---|
45 | #region parameter properties
|
---|
46 | public ILookupParameter<DataTable> VariableFrequenciesParameter {
|
---|
47 | get { return (ILookupParameter<DataTable>)Parameters[VariableFrequenciesParameterName]; }
|
---|
48 | }
|
---|
49 | public ILookupParameter<DoubleMatrix> VariableImpactsParameter {
|
---|
50 | get { return (ILookupParameter<DoubleMatrix>)Parameters[VariableImpactsParameterName]; }
|
---|
51 | }
|
---|
52 | public IValueLookupParameter<BoolValue> AggregateLaggedVariablesParameter {
|
---|
53 | get { return (IValueLookupParameter<BoolValue>)Parameters[AggregateLaggedVariablesParameterName]; }
|
---|
54 | }
|
---|
55 | #endregion
|
---|
56 | #region properties
|
---|
57 | public BoolValue AggregateLaggedVariables {
|
---|
58 | get { return AggregateLaggedVariablesParameter.ActualValue; }
|
---|
59 | set { AggregateLaggedVariablesParameter.Value = value; }
|
---|
60 | }
|
---|
61 | #endregion
|
---|
62 | [StorableConstructor]
|
---|
63 | private SymbolicDataAnalysisVariableFrequencyAnalyzer(bool deserializing) : base(deserializing) { }
|
---|
64 | private SymbolicDataAnalysisVariableFrequencyAnalyzer(SymbolicDataAnalysisVariableFrequencyAnalyzer original, Cloner cloner)
|
---|
65 | : base(original, cloner) {
|
---|
66 | }
|
---|
67 | public SymbolicDataAnalysisVariableFrequencyAnalyzer()
|
---|
68 | : base() {
|
---|
69 | Parameters.Add(new LookupParameter<DataTable>(VariableFrequenciesParameterName, "The relative variable reference frequencies aggregated over all trees in the population."));
|
---|
70 | Parameters.Add(new LookupParameter<DoubleMatrix>(VariableImpactsParameterName, "The relative variable relevance calculated as the average relative variable frequency over the whole run."));
|
---|
71 | Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateLaggedVariablesParameterName, "Switch that determines whether all references to a variable should be aggregated regardless of time-offsets. Turn off to analyze all variable references with different time offsets separately.", new BoolValue(true)));
|
---|
72 | }
|
---|
73 |
|
---|
74 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
75 | return new SymbolicDataAnalysisVariableFrequencyAnalyzer(this, cloner);
|
---|
76 | }
|
---|
77 |
|
---|
78 | public override IOperation Apply() {
|
---|
79 | ItemArray<ISymbolicExpressionTree> expressions = SymbolicExpressionTreeParameter.ActualValue;
|
---|
80 | ResultCollection results = ResultCollection;
|
---|
81 | DataTable datatable;
|
---|
82 | if (VariableFrequenciesParameter.ActualValue == null) {
|
---|
83 | datatable = new DataTable("Variable frequencies", "Relative frequency of variable references aggregated over the whole population.");
|
---|
84 | datatable.VisualProperties.XAxisTitle = "Generation";
|
---|
85 | datatable.VisualProperties.YAxisTitle = "Relative Variable Frequency";
|
---|
86 | VariableFrequenciesParameter.ActualValue = datatable;
|
---|
87 | results.Add(new Result("Variable frequencies", "Relative frequency of variable references aggregated over the whole population.", datatable));
|
---|
88 | results.Add(new Result("Variable impacts", "The relative variable relevance calculated as the average relative variable frequency over the whole run.", new DoubleMatrix()));
|
---|
89 | }
|
---|
90 |
|
---|
91 | datatable = VariableFrequenciesParameter.ActualValue;
|
---|
92 | // all rows must have the same number of values so we can just take the first
|
---|
93 | int numberOfValues = datatable.Rows.Select(r => r.Values.Count).DefaultIfEmpty().First();
|
---|
94 |
|
---|
95 | foreach (var pair in SymbolicDataAnalysisVariableFrequencyAnalyzer.CalculateVariableFrequencies(expressions, AggregateLaggedVariables.Value)) {
|
---|
96 | if (!datatable.Rows.ContainsKey(pair.Key)) {
|
---|
97 | // initialize a new row for the variable and pad with zeros
|
---|
98 | DataRow row = new DataRow(pair.Key, "", Enumerable.Repeat(0.0, numberOfValues));
|
---|
99 | row.VisualProperties.StartIndexZero = true;
|
---|
100 | datatable.Rows.Add(row);
|
---|
101 | }
|
---|
102 | datatable.Rows[pair.Key].Values.Add(Math.Round(pair.Value, 3));
|
---|
103 | }
|
---|
104 |
|
---|
105 | // add a zero for each data row that was not modified in the previous loop
|
---|
106 | foreach (var row in datatable.Rows.Where(r => r.Values.Count != numberOfValues + 1))
|
---|
107 | row.Values.Add(0.0);
|
---|
108 |
|
---|
109 | // update variable impacts matrix
|
---|
110 | var orderedImpacts = (from row in datatable.Rows
|
---|
111 | select new { Name = row.Name, Impact = Math.Round(datatable.Rows[row.Name].Values.Average(), 3) })
|
---|
112 | .OrderByDescending(p => p.Impact)
|
---|
113 | .ToList();
|
---|
114 | var impacts = new DoubleMatrix();
|
---|
115 | var matrix = impacts as IStringConvertibleMatrix;
|
---|
116 | matrix.Rows = orderedImpacts.Count;
|
---|
117 | matrix.RowNames = orderedImpacts.Select(x => x.Name);
|
---|
118 | matrix.Columns = 1;
|
---|
119 | matrix.ColumnNames = new string[] { "Relative variable relevance" };
|
---|
120 | int i = 0;
|
---|
121 | foreach (var p in orderedImpacts) {
|
---|
122 | matrix.SetValue(p.Impact.ToString(), i++, 0);
|
---|
123 | }
|
---|
124 |
|
---|
125 | VariableImpactsParameter.ActualValue = impacts;
|
---|
126 | results["Variable impacts"].Value = impacts;
|
---|
127 | return base.Apply();
|
---|
128 | }
|
---|
129 |
|
---|
130 | public static IEnumerable<KeyValuePair<string, double>> CalculateVariableFrequencies(IEnumerable<ISymbolicExpressionTree> trees, bool aggregateLaggedVariables = true) {
|
---|
131 |
|
---|
132 | var variableFrequencies = trees
|
---|
133 | .AsParallel()
|
---|
134 | .SelectMany(t => GetVariableReferences(t, aggregateLaggedVariables))
|
---|
135 | .GroupBy(pair => pair.Key, pair => pair.Value)
|
---|
136 | .ToDictionary(g => g.Key, g => (double)g.Sum());
|
---|
137 |
|
---|
138 | double totalNumberOfSymbols = variableFrequencies.Values.Sum();
|
---|
139 |
|
---|
140 | foreach (var pair in variableFrequencies.OrderBy(p => p.Key, new NaturalStringComparer()))
|
---|
141 | yield return new KeyValuePair<string, double>(pair.Key, pair.Value / totalNumberOfSymbols);
|
---|
142 | }
|
---|
143 |
|
---|
144 | private static IEnumerable<KeyValuePair<string, int>> GetVariableReferences(ISymbolicExpressionTree tree, bool aggregateLaggedVariables = true) {
|
---|
145 | Dictionary<string, int> references = new Dictionary<string, int>();
|
---|
146 | if (aggregateLaggedVariables) {
|
---|
147 | tree.Root.ForEachNodePrefix(node => {
|
---|
148 | if (node.Symbol is Variable) {
|
---|
149 | var varNode = node as VariableTreeNode;
|
---|
150 | IncReferenceCount(references, varNode.VariableName);
|
---|
151 | } else if (node.Symbol is VariableCondition) {
|
---|
152 | var varCondNode = node as VariableConditionTreeNode;
|
---|
153 | IncReferenceCount(references, varCondNode.VariableName);
|
---|
154 | }
|
---|
155 | });
|
---|
156 | } else {
|
---|
157 | GetVariableReferences(references, tree.Root, 0);
|
---|
158 | }
|
---|
159 | return references;
|
---|
160 | }
|
---|
161 |
|
---|
162 | private static void GetVariableReferences(Dictionary<string, int> references, ISymbolicExpressionTreeNode node, int currentLag) {
|
---|
163 | if (node.Symbol is LaggedVariable) {
|
---|
164 | var laggedVarNode = node as LaggedVariableTreeNode;
|
---|
165 | IncReferenceCount(references, laggedVarNode.VariableName, currentLag + laggedVarNode.Lag);
|
---|
166 | } else if (node.Symbol is Variable) {
|
---|
167 | var varNode = node as VariableTreeNode;
|
---|
168 | IncReferenceCount(references, varNode.VariableName, currentLag);
|
---|
169 | } else if (node.Symbol is VariableCondition) {
|
---|
170 | var varCondNode = node as VariableConditionTreeNode;
|
---|
171 | IncReferenceCount(references, varCondNode.VariableName, currentLag);
|
---|
172 | GetVariableReferences(references, node.GetSubtree(0), currentLag);
|
---|
173 | GetVariableReferences(references, node.GetSubtree(1), currentLag);
|
---|
174 | } else if (node.Symbol is Integral) {
|
---|
175 | var laggedNode = node as LaggedTreeNode;
|
---|
176 | for (int l = laggedNode.Lag; l <= 0; l++) {
|
---|
177 | GetVariableReferences(references, node.GetSubtree(0), currentLag + l);
|
---|
178 | }
|
---|
179 | } else if (node.Symbol is Derivative) {
|
---|
180 | for (int l = -4; l <= 0; l++) {
|
---|
181 | GetVariableReferences(references, node.GetSubtree(0), currentLag + l);
|
---|
182 | }
|
---|
183 | } else if (node.Symbol is TimeLag) {
|
---|
184 | var laggedNode = node as LaggedTreeNode;
|
---|
185 | GetVariableReferences(references, node.GetSubtree(0), currentLag + laggedNode.Lag);
|
---|
186 | } else {
|
---|
187 | foreach (var subtree in node.Subtrees) {
|
---|
188 | GetVariableReferences(references, subtree, currentLag);
|
---|
189 | }
|
---|
190 | }
|
---|
191 | }
|
---|
192 |
|
---|
193 | private static void IncReferenceCount(Dictionary<string, int> references, string variableName, int timeLag = 0) {
|
---|
194 | string referenceId = variableName +
|
---|
195 | (timeLag == 0 ? "" : timeLag < 0 ? "(t" + timeLag + ")" : "(t+" + timeLag + ")");
|
---|
196 | if (references.ContainsKey(referenceId)) {
|
---|
197 | references[referenceId]++;
|
---|
198 | } else {
|
---|
199 | references[referenceId] = 1;
|
---|
200 | }
|
---|
201 | }
|
---|
202 | }
|
---|
203 | }
|
---|