Changeset 3531 for trunk/sources

Timestamp:

04/26/10 11:05:28 (15 years ago)

Author:

gkronber

Message:

Added operator for calculation of relative variable frequencies. #938 (Data types and operators for regression problems)

Location:

trunk/sources

Files:

• HeuristicLab.Problems.DataAnalysis.Regression/3.3/HeuristicLab.Problems.DataAnalysis.Regression-3.3.csproj (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/BestValidationSymbolicRegressionSolutionVisualizer.cs (modified) (8 diffs)
• HeuristicLab.Problems.DataAnalysis/3.3/HeuristicLab.Problems.DataAnalysis-3.3.csproj (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis/3.3/Symbolic/VariableFrequencyAnalyser.cs (copied) (copied from trunk/sources/HeuristicLab.GP.StructureIdentification/3.3/VariableFrequencyAnalyser.cs) (2 diffs)

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/HeuristicLab.Problems.DataAnalysis.Regression-3.3.csproj

@@ -103 +103 @@
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
+      <Project>{887425B4-4348-49ED-A457-B7D2C26DDBF9}</Project>
+      <Name>HeuristicLab.Analysis-3.3</Name>
+    </ProjectReference>
     <ProjectReference Include="..\..\HeuristicLab.Collections\3.3\HeuristicLab.Collections-3.3.csproj">
       <Project>{958B43BC-CC5C-4FA2-8628-2B3B01D890B6}</Project>

trunk/sources/HeuristicLab.Problems.DataAnalysis.Regression/3.3/Symbolic/BestValidationSymbolicRegressionSolutionVisualizer.cs

@@ -31 +31 @@
 using HeuristicLab.Problems.DataAnalysis.Evaluators;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using System.Collections.Generic;
+using HeuristicLab.Analysis;
 
 namespace HeuristicLab.Problems.DataAnalysis.Regression.Symbolic {
@@ -49 +51 @@
     private const string QualityParameterName = "Quality";
     private const string ResultsParameterName = "Results";
+    private const string VariableFrequenciesParameterName = "VariableFrequencies";
 
     #region parameter properties
@@ -87 +90 @@
       get { return (ILookupParameter<ResultCollection>)Parameters[ResultsParameterName]; }
     }
+    public ILookupParameter<DataTable> VariableFrequenciesParameter {
+      get { return (ILookupParameter<DataTable>)Parameters[VariableFrequenciesParameterName]; }
+    }
+
     #endregion
 
@@ -104 +111 @@
     public IntValue ValidationSamplesEnd {
       get { return ValidationSamplesEndParameter.ActualValue; }
+    }
+    public DataTable VariableFrequencies {
+      get { return VariableFrequenciesParameter.ActualValue; }
+      set { VariableFrequenciesParameter.ActualValue = value; }
     }
     #endregion
@@ -118 +129 @@
       Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesEndParameterName, "The end index of the validation partition (part of the training partition)."));
       Parameters.Add(new LookupParameter<SymbolicRegressionSolution>(BestValidationSolutionParameterName, "The best symbolic expression tree based on the validation data for the symbolic regression problem."));
+      Parameters.Add(new LookupParameter<DataTable>(VariableFrequenciesParameterName, "The relative variable reference frequencies aggregated over the whole population."));
       Parameters.Add(new LookupParameter<ResultCollection>(ResultsParameterName, "The result collection of the algorithm."));
     }
@@ -124 +136 @@
       ItemArray<SymbolicExpressionTree> expressions = SymbolicExpressionTreeParameter.ActualValue;
       DataAnalysisProblemData problemData = DataAnalysisProblemDataParameter.ActualValue;
-
+      #region update variable frequencies
+      var inputVariables = problemData.InputVariables.Select(x => x.Value);
+      if (VariableFrequencies == null) {
+        VariableFrequencies = new DataTable("Variable Frequencies", "Relative frequency of variable references aggregated over the whole population.");
+        AddResult("VariableFrequencies", VariableFrequencies);
+        // add a data row for each input variable
+        foreach (var inputVariable in inputVariables)
+          VariableFrequencies.Rows.Add(new DataRow(inputVariable));
+      }
+      foreach (var pair in VariableFrequencyAnalyser.CalculateVariableFrequencies(expressions, inputVariables)) {
+        VariableFrequencies.Rows[pair.Key].Values.Add(pair.Value);
+      }
+      #endregion
+
+      #region determination of validation-best solution
       int validationSamplesStart = ValidationSamplesStart.Value;
       int validationSamplesEnd = ValidationSamplesEnd.Value;
@@ -142 +168 @@
 
       SymbolicRegressionSolution bestOfRunSolution = BestValidationSolutionParameter.ActualValue;
+      #endregion
+      #region update of validation-best solution
       if (bestOfRunSolution == null) {
         // no best of run solution yet -> make a solution from the currentBestExpression
@@ -153 +181 @@
         }
       }
-
-
+      #endregion
       return base.Apply();
     }

trunk/sources/HeuristicLab.Problems.DataAnalysis/3.3/HeuristicLab.Problems.DataAnalysis-3.3.csproj

@@ -112 +112 @@
     <Compile Include="Symbolic\Symbols\Variable.cs" />
     <Compile Include="Symbolic\Symbols\VariableTreeNode.cs" />
+    <Compile Include="Symbolic\VariableFrequencyAnalyser.cs" />
   </ItemGroup>
   <ItemGroup>

trunk/sources/HeuristicLab.Problems.DataAnalysis/3.3/Symbolic/VariableFrequencyAnalyser.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2008 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -24 +24 @@
 using HeuristicLab.Core;
 using HeuristicLab.Data;
-using HeuristicLab.GP.Interfaces;
 using System;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Operators;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Symbols;
 
-namespace HeuristicLab.GP.StructureIdentification {
-  /// <summary>
-  /// Creates accumulated frequencies of variable-symbols over the whole population.
-  /// </summary>
-  public class VariableFrequencyAnalyser : OperatorBase {
-    public override string Description {
-      get {
-        return @"Creates accumulated frequencies of variable-symbols over the whole population.";
-      }
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
+  [Item("VariableFrequencyAnalyser", "Calculates the accumulated frequencies of variable-symbols over the whole population.")]
+  [StorableClass]
+  public abstract class VariableFrequencyAnalyser : SingleSuccessorOperator {
+    private const string SymbolicExpressionTreeParameterName = "SymbolicExpressionTree";
+    private const string DataAnalysisProblemDataParameterName = "DataAnalysisProblemData";
+    private const string VariableFrequenciesParameterName = "VariableFrequencies";
+
+    #region parameter properties
+    public ILookupParameter<DataAnalysisProblemData> DataAnalysisProblemDataParameter {
+      get { return (ILookupParameter<DataAnalysisProblemData>)Parameters[DataAnalysisProblemDataParameterName]; }
     }
+    public ILookupParameter<ItemArray<SymbolicExpressionTree>> SymbolicExpressionTreeParameter {
+      get { return (ILookupParameter<ItemArray<SymbolicExpressionTree>>)Parameters[SymbolicExpressionTreeParameterName]; }
+    }
+    public ILookupParameter<DoubleMatrix> VariableFrequenciesParameter {
+      get { return (ILookupParameter<DoubleMatrix>)Parameters[VariableFrequenciesParameterName]; }
+    }
+    #endregion
+    #region properties
+    public DataAnalysisProblemData DataAnalysisProblemData {
+      get { return DataAnalysisProblemDataParameter.ActualValue; }
+    }
+    public ItemArray<SymbolicExpressionTree> SymbolicExpressionTrees {
+      get { return SymbolicExpressionTreeParameter.ActualValue; }
+    }
+    public DoubleMatrix VariableFrequencies {
+      get { return VariableFrequenciesParameter.ActualValue; }
+      set { VariableFrequenciesParameter.ActualValue = value; }
+    }
+    #endregion
     public VariableFrequencyAnalyser()
       : base() {
-      AddVariableInfo(new VariableInfo("InputVariables", "The input variables", typeof(ItemList), VariableKind.In));
-      AddVariableInfo(new VariableInfo("FunctionTree", "The tree to analyse", typeof(IGeneticProgrammingModel), VariableKind.In));
-      AddVariableInfo(new VariableInfo("VariableFrequency", "The accumulated variable-frequencies over the whole population.", typeof(ItemList<ItemList>), VariableKind.New | VariableKind.Out));
+      Parameters.Add(new SubScopesLookupParameter<SymbolicExpressionTree>(SymbolicExpressionTreeParameterName, "The symbolic expression trees that should be analyzed."));
+      Parameters.Add(new LookupParameter<DataAnalysisProblemData>(DataAnalysisProblemDataParameterName, "The problem data on which the for which the symbolic expression tree is a solution."));
+      Parameters.Add(new LookupParameter<DoubleMatrix>(VariableFrequenciesParameterName, "The relative variable reference frequencies aggregated over the whole population."));
     }
 
-    public override IOperation Apply(IScope scope) {
-      ItemList<ItemList> frequenciesList = GetVariableValue<ItemList<ItemList>>("VariableFrequency", scope, false, false);
-      ItemList inputVariables = GetVariableValue<ItemList>("InputVariables", scope, true);
-      if (frequenciesList == null) {
-        frequenciesList = new ItemList<ItemList>();
-        // first line should contain a list of variables
-        ItemList varList = new ItemList();
-        foreach (var inputVariable in inputVariables) {
-          varList.Add(inputVariable);
-        }
-        frequenciesList.Add(varList);
-        IVariableInfo info = GetVariableInfo("VariableFrequency");
-        if (info.Local)
-          AddVariable(new HeuristicLab.Core.Variable(info.ActualName, frequenciesList));
-        else
-          scope.AddVariable(new HeuristicLab.Core.Variable(scope.TranslateName(info.FormalName), frequenciesList));
+    public override IOperation Apply() {
+      var inputVariables = DataAnalysisProblemData.InputVariables.Select(x => x.Value);
+      if (VariableFrequencies == null) {
+        VariableFrequencies = new DoubleMatrix(0, 1, inputVariables);
       }
-      double[] frequencySum = new double[inputVariables.Count()];
-      int variableNodesSum = 0;
-      foreach (var subScope in scope.SubScopes) {
-        IGeneticProgrammingModel gpModel = GetVariableValue<IGeneticProgrammingModel>("FunctionTree", subScope, false);
-        var subScopeFrequencies = GetFrequencies(gpModel.FunctionTree, inputVariables);
-        if (subScopeFrequencies.Count() != frequencySum.Length) throw new InvalidProgramException();
-        int i = 0;
-        foreach (var freq in subScopeFrequencies) {
-          frequencySum[i++] += freq;
-        }
-        variableNodesSum += CountVariableNodes(gpModel.FunctionTree);
+      ((IStringConvertibleMatrix)VariableFrequencies).Rows = VariableFrequencies.Rows + 1;
+      int lastRowIndex = VariableFrequencies.Rows - 1;
+      var columnNames = VariableFrequencies.ColumnNames.ToList();
+      foreach (var pair in CalculateVariableFrequencies(SymbolicExpressionTrees, inputVariables)) {
+        int columnIndex = columnNames.IndexOf(pair.Key);
+        VariableFrequencies[lastRowIndex, columnIndex] = pair.Value;
       }
-      ItemList freqList = new ItemList();
-      for (int i = 0; i < frequencySum.Length; i++) {
-        freqList.Add(new DoubleData(frequencySum[i] / variableNodesSum));
-      }
-      frequenciesList.Add(freqList);
       return null;
     }
 
-    private int CountVariableNodes(IFunctionTree tree) {
-      return (from x in FunctionTreeIterator.IteratePostfix(tree)
-              where x is VariableFunctionTree
-              select 1).Sum();
+    public static IEnumerable<KeyValuePair<string, double>> CalculateVariableFrequencies(IEnumerable<SymbolicExpressionTree> trees, IEnumerable<string> inputVariables) {
+      int totalVariableReferences = 0;
+      Dictionary<string, double> variableReferencesSum = new Dictionary<string, double>();
+      foreach (var inputVariable in inputVariables)
+        variableReferencesSum[inputVariable] = 0.0;
+      foreach (var tree in trees) {
+        var variableReferences = GetVariableReferenceCount(tree, inputVariables);
+        foreach (var pair in variableReferences) {
+          variableReferencesSum[pair.Key] += pair.Value;
+        }
+        totalVariableReferences += GetTotalVariableReferencesCount(tree);
+      }
+      foreach (string inputVariable in inputVariables) {
+        double relFreq = variableReferencesSum[inputVariable] / (double)totalVariableReferences;
+        yield return new KeyValuePair<string, double>(inputVariable, relFreq);
+      }
     }
 
-    private static IEnumerable<double> GetFrequencies(IFunctionTree tree, ItemList inputVariables) {
-      var groupedFuns = (from node in FunctionTreeIterator.IteratePostfix(tree)
-                         let varNode = node as VariableFunctionTree
-                         where varNode != null
-                         select varNode.VariableName).GroupBy(x => x);
+    private static int GetTotalVariableReferencesCount(SymbolicExpressionTree tree) {
+      return tree.IterateNodesPrefix().OfType<VariableTreeNode>().Count();
+    }
 
-      foreach (var inputVariable in inputVariables.Cast<StringData>()) {
+    private static IEnumerable<KeyValuePair<string, int>> GetVariableReferenceCount(SymbolicExpressionTree tree, IEnumerable<string> inputVariables) {
+      var groupedFuns = (from node in tree.IterateNodesPrefix().OfType<VariableTreeNode>()
+                         select node.VariableName).GroupBy(x => x);
+
+      foreach (var inputVariable in inputVariables) {
         var matchingFuns = from g in groupedFuns
-                           where g.Key == inputVariable.Data
+                           where g.Key == inputVariable
                            select g.Count();
-        if (matchingFuns.Count() == 0) yield return 0.0;
+        if (matchingFuns.Count() == 0) yield return new KeyValuePair<string, int>(inputVariable, 0);
         else {
-          yield return matchingFuns.Single(); // / (double)gpModel.Size;
+          yield return new KeyValuePair<string, int>(inputVariable, matchingFuns.Single());
         }
       }