Changeset 14232 for branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic

Timestamp:

08/03/16 18:54:14 (8 years ago)

Author:

gkronber

Message:

created a feature branch for #2650 (support for categorical variables in symb reg) with a first set of changes

work in progress...

Location:

branches/symbreg-factors-2650

Files:

• . (copied) (copied from trunk/sources)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers/SymbolicDataAnalysisVariableFrequencyAnalyzer.cs (modified) (10 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/ArithmeticExpressionGrammar.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/FullFunctionalExpressionGrammar.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/TypeCoherentExpressionGrammar.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interfaces/ISymbolicDataAnalysisImpactValuesCalculator.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/OpCodes.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeILEmittingInterpreter.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeInterpreter.cs (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeLinearInterpreter.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisModelComplexityCalculator.cs (modified) (4 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisSolutionImpactValuesCalculator.cs (modified) (2 diffs)

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Analyzers/SymbolicDataAnalysisVariableFrequencyAnalyzer.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using HeuristicLab.Analysis;
@@ -41 +42 @@
     private const string VariableFrequenciesParameterName = "VariableFrequencies";
     private const string AggregateLaggedVariablesParameterName = "AggregateLaggedVariables";
+    private const string AggregateFactorVariablesParameterName = "AggregateFactorVariables";
     private const string VariableImpactsParameterName = "VariableImpacts";
 
@@ -52 +54 @@
     public IValueLookupParameter<BoolValue> AggregateLaggedVariablesParameter {
       get { return (IValueLookupParameter<BoolValue>)Parameters[AggregateLaggedVariablesParameterName]; }
+    }
+    public IValueLookupParameter<BoolValue> AggregateFactorVariablesParameter {
+      get { return (IValueLookupParameter<BoolValue>)Parameters[AggregateFactorVariablesParameterName]; }
     }
     #endregion
@@ -59 +64 @@
       set { AggregateLaggedVariablesParameter.Value = value; }
     }
+    public BoolValue AggregateFactorVariables {
+      get { return AggregateFactorVariablesParameter.ActualValue; }
+      set { AggregateFactorVariablesParameter.Value = value; }
+    }
     #endregion
     [StorableConstructor]
@@ -70 +79 @@
       Parameters.Add(new LookupParameter<DoubleMatrix>(VariableImpactsParameterName, "The relative variable relevance calculated as the average relative variable frequency over the whole run."));
       Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateLaggedVariablesParameterName, "Switch that determines whether all references to a variable should be aggregated regardless of time-offsets. Turn off to analyze all variable references with different time offsets separately.", new BoolValue(true)));
+      Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateFactorVariablesParameterName, "Switch that determines whether all references to factor variables should be aggregated regardless of the value. Turn off to analyze all factor variable references with different values separately.", new BoolValue(true)));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+      // BackwardsCompatibility3.3
+      #region Backwards compatible code, remove with 3.4
+      if (!Parameters.ContainsKey(AggregateFactorVariablesParameterName)) {
+        Parameters.Add(new ValueLookupParameter<BoolValue>(AggregateFactorVariablesParameterName, "Switch that determines whether all references to factor variables should be aggregated regardless of the value. Turn off to analyze all factor variable references with different values separately.", new BoolValue(true)));
+      }
+      #endregion
     }
 
@@ -93 +113 @@
       int numberOfValues = datatable.Rows.Select(r => r.Values.Count).DefaultIfEmpty().First();
 
-      foreach (var pair in SymbolicDataAnalysisVariableFrequencyAnalyzer.CalculateVariableFrequencies(expressions, AggregateLaggedVariables.Value)) {
+      foreach (var pair in CalculateVariableFrequencies(expressions, AggregateLaggedVariables.Value, AggregateFactorVariables.Value)) {
         if (!datatable.Rows.ContainsKey(pair.Key)) {
           // initialize a new row for the variable and pad with zeros
@@ -128 +148 @@
     }
 
-    public static IEnumerable<KeyValuePair<string, double>> CalculateVariableFrequencies(IEnumerable<ISymbolicExpressionTree> trees, bool aggregateLaggedVariables = true) {
+    public static IEnumerable<KeyValuePair<string, double>> CalculateVariableFrequencies(IEnumerable<ISymbolicExpressionTree> trees,
+      bool aggregateLaggedVariables = true, bool aggregateFactorVariables = true) {
 
       var variableFrequencies = trees
-        .SelectMany(t => GetVariableReferences(t, aggregateLaggedVariables))
+        .SelectMany(t => GetVariableReferences(t, aggregateLaggedVariables, aggregateFactorVariables))
         .GroupBy(pair => pair.Key, pair => pair.Value)
         .ToDictionary(g => g.Key, g => (double)g.Sum());
@@ -141 +162 @@
     }
 
-    private static IEnumerable<KeyValuePair<string, int>> GetVariableReferences(ISymbolicExpressionTree tree, bool aggregateLaggedVariables = true) {
+    private static IEnumerable<KeyValuePair<string, int>> GetVariableReferences(ISymbolicExpressionTree tree,
+      bool aggregateLaggedVariables = true, bool aggregateFactorVariables = true) {
       Dictionary<string, int> references = new Dictionary<string, int>();
       if (aggregateLaggedVariables) {
@@ -151 +173 @@
             var varCondNode = node as VariableConditionTreeNode;
             IncReferenceCount(references, varCondNode.VariableName);
+          } else if (node.Symbol is FactorVariable) {
+            var factorNode = node as FactorVariableTreeNode;
+            if (aggregateFactorVariables) {
+              IncReferenceCount(references, factorNode.VariableName);
+            } else {
+              IncReferenceCount(references, factorNode.ToString());
+            }
           }
         });
       } else {
-        GetVariableReferences(references, tree.Root, 0);
+        GetVariableReferences(references, tree.Root, 0, aggregateFactorVariables);
       }
       return references;
     }
 
-    private static void GetVariableReferences(Dictionary<string, int> references, ISymbolicExpressionTreeNode node, int currentLag) {
+    private static void GetVariableReferences(Dictionary<string, int> references, ISymbolicExpressionTreeNode node, int currentLag, bool aggregateFactorVariables) {
       if (node.Symbol is LaggedVariable) {
         var laggedVarNode = node as LaggedVariableTreeNode;
@@ -166 +195 @@
         var varNode = node as VariableTreeNode;
         IncReferenceCount(references, varNode.VariableName, currentLag);
+      } else if (node.Symbol is FactorVariable) {
+        var factorNode = node as FactorVariableTreeNode;
+        if (aggregateFactorVariables) {
+          IncReferenceCount(references, factorNode.VariableName, currentLag);
+        } else {
+          IncReferenceCount(references, factorNode.ToString(), currentLag);
+        }
       } else if (node.Symbol is VariableCondition) {
         var varCondNode = node as VariableConditionTreeNode;
         IncReferenceCount(references, varCondNode.VariableName, currentLag);
-        GetVariableReferences(references, node.GetSubtree(0), currentLag);
-        GetVariableReferences(references, node.GetSubtree(1), currentLag);
+        GetVariableReferences(references, node.GetSubtree(0), currentLag, aggregateFactorVariables);
+        GetVariableReferences(references, node.GetSubtree(1), currentLag, aggregateFactorVariables);
       } else if (node.Symbol is Integral) {
         var laggedNode = node as LaggedTreeNode;
         for (int l = laggedNode.Lag; l <= 0; l++) {
-          GetVariableReferences(references, node.GetSubtree(0), currentLag + l);
+          GetVariableReferences(references, node.GetSubtree(0), currentLag + l, aggregateFactorVariables);
         }
       } else if (node.Symbol is Derivative) {
         for (int l = -4; l <= 0; l++) {
-          GetVariableReferences(references, node.GetSubtree(0), currentLag + l);
+          GetVariableReferences(references, node.GetSubtree(0), currentLag + l, aggregateFactorVariables);
         }
       } else if (node.Symbol is TimeLag) {
         var laggedNode = node as LaggedTreeNode;
-        GetVariableReferences(references, node.GetSubtree(0), currentLag + laggedNode.Lag);
+        GetVariableReferences(references, node.GetSubtree(0), currentLag + laggedNode.Lag, aggregateFactorVariables);
       } else {
         foreach (var subtree in node.Subtrees) {
-          GetVariableReferences(references, subtree, currentLag);
+          GetVariableReferences(references, subtree, currentLag, aggregateFactorVariables);
         }
       }

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/ArithmeticExpressionGrammar.cs

@@ -53 +53 @@
       constant.MaxValue = 20;
       var variableSymbol = new HeuristicLab.Problems.DataAnalysis.Symbolic.Variable();
+      var factorVariableSymbol = new FactorVariable();
 
-      var allSymbols = new List<Symbol>() { add, sub, mul, div, constant, variableSymbol };
+      var allSymbols = new List<Symbol>() { add, sub, mul, div, constant, variableSymbol, factorVariableSymbol };
       var functionSymbols = new List<Symbol>() { add, sub, mul, div };
 
@@ -65 +66 @@
       SetSubtreeCount(constant, 0, 0);
       SetSubtreeCount(variableSymbol, 0, 0);
+      SetSubtreeCount(factorVariableSymbol, 0, 0);
 
       // allow each symbol as child of the start symbol

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/FullFunctionalExpressionGrammar.cs

@@ -115 +115 @@
       constant.MaxValue = 20;
       var variableSymbol = new HeuristicLab.Problems.DataAnalysis.Symbolic.Variable();
+      var factorVariable = new FactorVariable();
       var laggedVariable = new LaggedVariable();
       laggedVariable.InitialFrequency = 0.0;
@@ -123 +124 @@
       var allSymbols = new List<Symbol>() { add, sub, mul, div, mean, sin, cos, tan, log, square, pow, sqrt, root, exp,
         airyA, airyB, bessel, cosineIntegral, dawson, erf, expIntegralEi, fresnelCosineIntegral, fresnelSineIntegral, gamma, hypCosineIntegral, hypSineIntegral, norm, psi, sineIntegral,
-        @if, gt, lt, and, or, not,xor, timeLag, integral, derivative, constant, variableSymbol, laggedVariable,autoregressiveVariable, variableCondition };
+        @if, gt, lt, and, or, not,xor, timeLag, integral, derivative, constant, variableSymbol, factorVariable, laggedVariable,autoregressiveVariable, variableCondition };
       var unaryFunctionSymbols = new List<Symbol>() { square, sqrt, sin, cos, tan, log, exp, not, timeLag, integral, derivative,
         airyA, airyB, bessel, cosineIntegral, dawson, erf, expIntegralEi, fresnelCosineIntegral, fresnelSineIntegral, gamma, hypCosineIntegral, hypSineIntegral, norm, psi, sineIntegral
@@ -130 +131 @@
       var binaryFunctionSymbols = new List<Symbol>() { pow, root, gt, lt, variableCondition };
       var ternarySymbols = new List<Symbol>() { add, sub, mul, div, mean, and, or, xor };
-      var terminalSymbols = new List<Symbol>() { variableSymbol, constant, laggedVariable, autoregressiveVariable };
+      var terminalSymbols = new List<Symbol>() { variableSymbol, factorVariable, constant, laggedVariable, autoregressiveVariable };
 
       foreach (var symb in allSymbols)

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/TypeCoherentExpressionGrammar.cs

@@ -104 +104 @@
       constant.MaxValue = 20;
       var variableSymbol = new Variable();
+      var factorVariable = new FactorVariable();
       var laggedVariable = new LaggedVariable();
       var autoregressiveVariable = new AutoregressiveTargetVariable();
@@ -114 +115 @@
       var specialFunctions = new GroupSymbol(SpecialFunctionsName, new List<ISymbol> { airyA, airyB, bessel, cosineIntegral, dawson, erf, expIntegralEi, 
         fresnelCosineIntegral,fresnelSineIntegral,gamma,hypCosineIntegral,hypSineIntegral,norm, psi, sineIntegral});
-      var terminalSymbols = new GroupSymbol(TerminalsName, new List<ISymbol> { constant, variableSymbol });
+      var terminalSymbols = new GroupSymbol(TerminalsName, new List<ISymbol> { constant, variableSymbol, factorVariable });
       var realValuedSymbols = new GroupSymbol(RealValuedSymbolsName, new List<ISymbol>() { arithmeticSymbols, trigonometricSymbols, exponentialAndLogarithmicSymbols, specialFunctions, terminalSymbols });
 
@@ -122 +123 @@
       var comparisonSymbols = new GroupSymbol(ComparisonsName, new List<ISymbol> { gt, lt });
       var booleanOperationSymbols = new GroupSymbol(BooleanOperatorsName, new List<ISymbol> { and, or, not, xor });
-      var conditionalSymbols = new GroupSymbol(ConditionalSymbolsName, new List<ISymbol> { conditionSymbols, comparisonSymbols, booleanOperationSymbols });
+      var conditionalSymbols = new GroupSymbol(ConditionalSymbolsName, new List<ISymbol> { conditionSymbols, comparisonSymbols, booleanOperationSymbols }); // TODO: factorVariableBool?
 
       var timeSeriesSymbols = new GroupSymbol(TimeSeriesSymbolsName, new List<ISymbol> { timeLag, integral, derivative, laggedVariable, autoregressiveVariable });

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj

@@ -198 +198 @@
     <Compile Include="Symbols\AiryB.cs" />
     <Compile Include="Symbols\Bessel.cs" />
+    <Compile Include="Symbols\FactorVariable.cs" />
+    <Compile Include="Symbols\FactorVariableTreeNode.cs" />
     <Compile Include="Symbols\Xor.cs" />
     <Compile Include="Symbols\Erf.cs" />

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interfaces/ISymbolicDataAnalysisImpactValuesCalculator.cs

@@ -5 +5 @@
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
   public interface ISymbolicDataAnalysisSolutionImpactValuesCalculator : IItem {
-    double CalculateReplacementValue(ISymbolicDataAnalysisModel model, ISymbolicExpressionTreeNode node, IDataAnalysisProblemData problemData, IEnumerable<int> rows);
-    double CalculateImpactValue(ISymbolicDataAnalysisModel model, ISymbolicExpressionTreeNode node, IDataAnalysisProblemData problemData, IEnumerable<int> rows, double qualityForImpactsCalculation = double.NaN);
     void CalculateImpactAndReplacementValues(ISymbolicDataAnalysisModel model, ISymbolicExpressionTreeNode node, IDataAnalysisProblemData problemData,
       IEnumerable<int> rows, out double impactValue, out double replacementValue, out double newQualityForImpactsCalculation, double qualityForImpactsCalculation = double.NaN);

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/OpCodes.cs

@@ -83 +83 @@
     public const byte Erf = 43;
     public const byte Bessel = 44;
+    public const byte FactorVariable = 46;
 
     private static Dictionary<Type, byte> symbolToOpcode = new Dictionary<Type, byte>() {
@@ -130 +131 @@
       { typeof(Norm), OpCodes.Norm},
       { typeof(Erf), OpCodes.Erf},
-      { typeof(Bessel), OpCodes.Bessel}   
+      { typeof(Bessel), OpCodes.Bessel},
+      { typeof(FactorVariable), OpCodes.FactorVariable }
     };
 

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeILEmittingInterpreter.cs

@@ -66 +66 @@
     private static MethodInfo erf = thisType.GetMethod("Erf", new Type[] { typeof(double) });
     private static MethodInfo bessel = thisType.GetMethod("Bessel", new Type[] { typeof(double) });
+    private static MethodInfo string_eq = typeof(string).GetMethod("Equals", new Type[] {typeof(string)});
     #endregion
 
@@ -627 +628 @@
             return;
           }
+        case OpCodes.FactorVariable: {
+            FactorVariableTreeNode varNode = currentInstr.dynamicNode as FactorVariableTreeNode;
+            il.Emit(System.Reflection.Emit.OpCodes.Ldarg_1); // load columns array
+            il.Emit(System.Reflection.Emit.OpCodes.Ldc_I4, (int)currentInstr.data);
+            // load correct column of the current variable
+            il.Emit(System.Reflection.Emit.OpCodes.Ldelem_Ref);
+            il.Emit(System.Reflection.Emit.OpCodes.Ldarg_0); // rowIndex
+            if (!state.InLaggedContext) {
+              il.Emit(System.Reflection.Emit.OpCodes.Call, listGetValue);
+              il.Emit(System.Reflection.Emit.OpCodes.Ldc_R8, varNode.VariableValue); 
+              il.Emit(System.Reflection.Emit.OpCodes.Call, string_eq);
+              // TODO: convert bool to 1 / 0?
+            } else {
+              var nanResult = il.DefineLabel();
+              var normalResult = il.DefineLabel();
+              il.Emit(System.Reflection.Emit.OpCodes.Dup);
+              il.Emit(System.Reflection.Emit.OpCodes.Ldc_I4_0);
+              il.Emit(System.Reflection.Emit.OpCodes.Blt, nanResult);
+              il.Emit(System.Reflection.Emit.OpCodes.Dup);
+              il.Emit(System.Reflection.Emit.OpCodes.Ldc_I4, ds.Rows);
+              il.Emit(System.Reflection.Emit.OpCodes.Bge, nanResult);
+              il.Emit(System.Reflection.Emit.OpCodes.Call, listGetValue);
+              il.Emit(System.Reflection.Emit.OpCodes.Ldc_R8, varNode.VariableValue);
+              il.Emit(System.Reflection.Emit.OpCodes.Call, string_eq);
+              // TODO: convert bool to 1 / 0?
+              il.Emit(System.Reflection.Emit.OpCodes.Br, normalResult);
+              il.MarkLabel(nanResult);
+              il.Emit(System.Reflection.Emit.OpCodes.Pop); // rowIndex
+              il.Emit(System.Reflection.Emit.OpCodes.Pop); // column reference
+              il.Emit(System.Reflection.Emit.OpCodes.Ldc_R8, double.NaN);
+              il.MarkLabel(normalResult);
+            }
+            return;
+          }
         case OpCodes.LagVariable: {
             var nanResult = il.DefineLabel();

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeInterpreter.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -143 +144 @@
           var variableTreeNode = (VariableTreeNode)instr.dynamicNode;
           instr.data = dataset.GetReadOnlyDoubleValues(variableTreeNode.VariableName);
+        } else if (instr.opCode == OpCodes.FactorVariable) {
+          var factorTreeNode = instr.dynamicNode as FactorVariableTreeNode;
+          instr.data = dataset.GetReadOnlyStringValues(factorTreeNode.VariableName); 
         } else if (instr.opCode == OpCodes.LagVariable) {
           var laggedVariableTreeNode = (LaggedVariableTreeNode)instr.dynamicNode;
@@ -455 +459 @@
             return ((IList<double>)currentInstr.data)[row] * variableTreeNode.Weight;
           }
+        case OpCodes.FactorVariable: {
+            if (row < 0 || row >= dataset.Rows) return double.NaN;
+            var factorVarTreeNode = currentInstr.dynamicNode as FactorVariableTreeNode;
+            return ((IList<string>)currentInstr.data)[row] == factorVarTreeNode.VariableValue ? 1 : 0;
+          }
         case OpCodes.LagVariable: {
             var laggedVariableTreeNode = (LaggedVariableTreeNode)currentInstr.dynamicNode;

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeLinearInterpreter.cs

@@ -147 +147 @@
             var variableTreeNode = (VariableTreeNode)instr.dynamicNode;
             instr.value = ((IList<double>)instr.data)[row] * variableTreeNode.Weight;
+          }
+        } else if (instr.opCode == OpCodes.FactorVariable) {
+          if (row < 0 || row >= dataset.Rows) instr.value = double.NaN;
+          else {
+            var factorTreeNode = instr.dynamicNode as FactorVariableTreeNode;
+            instr.value = ((IList<string>)instr.data)[row] == factorTreeNode.VariableValue ? 1 : 0;
           }
         } else if (instr.opCode == OpCodes.LagVariable) {
@@ -392 +398 @@
             }
             break;
+          case OpCodes.FactorVariable: {
+              var factorVariableTreeNode = instr.dynamicNode as FactorVariableTreeNode;
+              instr.data = dataset.GetReadOnlyStringValues(factorVariableTreeNode.VariableName);
+            }
+            break;
           case OpCodes.LagVariable: {
               var laggedVariableTreeNode = (LaggedVariableTreeNode)instr.dynamicNode;

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisModelComplexityCalculator.cs

@@ -39 +39 @@
             return 1;
           }
-        case OpCodes.Variable: {
+        case OpCodes.Variable:
+        case OpCodes.FactorVariable: {
             return 2;
           }
-        case OpCodes.Add: 
+        case OpCodes.Add:
         case OpCodes.Sub: {
             double complexity = 0;
@@ -50 +51 @@
             return complexity;
           }
-        case OpCodes.Mul: 
+        case OpCodes.Mul:
         case OpCodes.Div: {
             double complexity = 1;
@@ -60 +61 @@
           }
         case OpCodes.Sin:
-        case OpCodes.Cos: 
+        case OpCodes.Cos:
         case OpCodes.Tan:
-        case OpCodes.Exp: 
+        case OpCodes.Exp:
         case OpCodes.Log: {
             double complexity = CalculateComplexity(node.GetSubtree(0));
@@ -75 +76 @@
             return complexity * complexity * complexity;
           }
-        case OpCodes.Power:          
+        case OpCodes.Power:
         case OpCodes.Root: {
             double complexity = CalculateComplexity(node.GetSubtree(0));

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisProblem.cs

@@ -208 +208 @@
 
     protected virtual void UpdateGrammar() {
-      SymbolicExpressionTreeGrammar.MaximumFunctionArguments = MaximumFunctionArguments.Value;
-      SymbolicExpressionTreeGrammar.MaximumFunctionDefinitions = MaximumFunctionDefinitions.Value;
-      foreach (var varSymbol in SymbolicExpressionTreeGrammar.Symbols.OfType<HeuristicLab.Problems.DataAnalysis.Symbolic.Variable>()) {
+      var problemData = ProblemData;
+      var ds = problemData.Dataset;
+      var grammar = SymbolicExpressionTreeGrammar;
+      grammar.MaximumFunctionArguments = MaximumFunctionArguments.Value;
+      grammar.MaximumFunctionDefinitions = MaximumFunctionDefinitions.Value;
+      foreach (var varSymbol in grammar.Symbols.OfType<HeuristicLab.Problems.DataAnalysis.Symbolic.Variable>()) {
         if (!varSymbol.Fixed) {
-          varSymbol.AllVariableNames = ProblemData.InputVariables.Select(x => x.Value);
-          varSymbol.VariableNames = ProblemData.AllowedInputVariables;
+          varSymbol.AllVariableNames = problemData.InputVariables.Select(x => x.Value).Where(x => ds.VariableHasType<double>(x));
+          varSymbol.VariableNames = problemData.AllowedInputVariables.Where(x => ds.VariableHasType<double>(x));
         }
       }
-      foreach (var varSymbol in SymbolicExpressionTreeGrammar.Symbols.OfType<HeuristicLab.Problems.DataAnalysis.Symbolic.VariableCondition>()) {
+      foreach (var factorSymbol in grammar.Symbols.OfType<FactorVariable>()) {
+        if (!factorSymbol.Fixed) {
+          factorSymbol.AllVariableNames = problemData.InputVariables.Select(x => x.Value).Where(x => ds.VariableHasType<string>(x));
+          factorSymbol.VariableNames = problemData.AllowedInputVariables.Where(x => ds.VariableHasType<string>(x));
+          factorSymbol.VariableValues = factorSymbol.VariableNames
+            .ToDictionary(varName => varName, varName => ds.GetStringValues(varName).Distinct().ToList());
+        }
+      }
+      foreach (var varSymbol in grammar.Symbols.OfType<HeuristicLab.Problems.DataAnalysis.Symbolic.VariableCondition>()) {
         if (!varSymbol.Fixed) {
-          varSymbol.AllVariableNames = ProblemData.InputVariables.Select(x => x.Value);
-          varSymbol.VariableNames = ProblemData.AllowedInputVariables;
+          varSymbol.AllVariableNames = problemData.InputVariables.Select(x => x.Value).Where(x => ds.VariableHasType<double>(x));
+          varSymbol.VariableNames = problemData.AllowedInputVariables;
         }
       }

branches/symbreg-factors-2650/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/SymbolicDataAnalysisSolutionImpactValuesCalculator.cs

@@ -21 +21 @@
 
 using System.Collections.Generic;
+using System.Linq;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
@@ -36 +37 @@
     [StorableConstructor]
     protected SymbolicDataAnalysisSolutionImpactValuesCalculator(bool deserializing) : base(deserializing) { }
-    public abstract double CalculateReplacementValue(ISymbolicDataAnalysisModel model, ISymbolicExpressionTreeNode node, IDataAnalysisProblemData problemData, IEnumerable<int> rows);
-    public abstract double CalculateImpactValue(ISymbolicDataAnalysisModel model, ISymbolicExpressionTreeNode node, IDataAnalysisProblemData problemData, IEnumerable<int> rows, double qualityForImpactsCalculation = double.NaN);
     public abstract void CalculateImpactAndReplacementValues(ISymbolicDataAnalysisModel model, ISymbolicExpressionTreeNode node, IDataAnalysisProblemData problemData, IEnumerable<int> rows, out double impactValue, out double replacementValue, out double newQualityForImpactsCalculation, double qualityForImpactsCalculation = double.NaN);
 
-    protected static double CalculateReplacementValue(ISymbolicExpressionTreeNode node, ISymbolicExpressionTree sourceTree, ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
+    protected IEnumerable<double> CalculateReplacementValues(ISymbolicExpressionTreeNode node, ISymbolicExpressionTree sourceTree, ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
       IDataset dataset, IEnumerable<int> rows) {
       //optimization: constant nodes return always the same value
       ConstantTreeNode constantNode = node as ConstantTreeNode;
-      if (constantNode != null) return constantNode.Value;
+      FactorVariableTreeNode factorNode = node as FactorVariableTreeNode;
+      if (constantNode != null) {
+        yield return constantNode.Value;
+      } else if (factorNode != null) {
+        // valid replacements are either all off or all on
+        yield return 0;
+        yield return 1;
+      } else {
+        var rootSymbol = new ProgramRootSymbol().CreateTreeNode();
+        var startSymbol = new StartSymbol().CreateTreeNode();
+        rootSymbol.AddSubtree(startSymbol);
+        startSymbol.AddSubtree((ISymbolicExpressionTreeNode)node.Clone());
 
-      var rootSymbol = new ProgramRootSymbol().CreateTreeNode();
-      var startSymbol = new StartSymbol().CreateTreeNode();
-      rootSymbol.AddSubtree(startSymbol);
-      startSymbol.AddSubtree((ISymbolicExpressionTreeNode)node.Clone());
-
-      var tempTree = new SymbolicExpressionTree(rootSymbol);
-      // clone ADFs of source tree
-      for (int i = 1; i < sourceTree.Root.SubtreeCount; i++) {
-        tempTree.Root.AddSubtree((ISymbolicExpressionTreeNode)sourceTree.Root.GetSubtree(i).Clone());
+        var tempTree = new SymbolicExpressionTree(rootSymbol);
+        // clone ADFs of source tree
+        for (int i = 1; i < sourceTree.Root.SubtreeCount; i++) {
+          tempTree.Root.AddSubtree((ISymbolicExpressionTreeNode)sourceTree.Root.GetSubtree(i).Clone());
+        }
+        yield return interpreter.GetSymbolicExpressionTreeValues(tempTree, dataset, rows).Median();
+        yield return interpreter.GetSymbolicExpressionTreeValues(tempTree, dataset, rows).Average(); // TODO perf
       }
-      return interpreter.GetSymbolicExpressionTreeValues(tempTree, dataset, rows).Median();
     }
   }