Changeset 12923 for branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies

Timestamp:

08/28/15 14:57:21 (9 years ago)

Author:

gkronber

Message:

#2471

• refactoring to use state value function V(s) instead of state/action value function Q(s,a)
• added test case for artificial ant problem

Location:

branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies

Files:

• BoltzmannExplorationSymbolicExpressionConstructionPolicy.cs (modified) (2 diffs)
• EpsGreedySymbolicExpressionConstructionPolicy.cs (modified) (2 diffs)
• RandomSymbolicExpressionConstructionPolicy.cs (modified) (1 diff)
• SymbolicExpressionConstructionPolicyBase.cs (modified) (6 diffs)
• UcbSymbolicExpressionConstructionPolicy.cs (modified) (2 diffs)
• UcbTunedSymbolicExpressionConstructionPolicy.cs (modified) (2 diffs)

branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies/BoltzmannExplorationSymbolicExpressionConstructionPolicy.cs

@@ -20 +20 @@
     }
 
-    public ITabularQualityFunction QualityFunction {
+    public ITabularStateValueFunction StateValueFunction {
       get {
-        return ((IValueParameter<ITabularQualityFunction>)Parameters["Quality function"]).Value;
+        return ((IValueParameter<ITabularStateValueFunction>)Parameters["Quality function"]).Value;
       }
-      set { ((IValueParameter<ITabularQualityFunction>)Parameters["Quality function"]).Value = value; }
+      set { ((IValueParameter<ITabularStateValueFunction>)Parameters["Quality function"]).Value = value; }
     }
 
@@ -39 +39 @@
       : base() {
       Parameters.Add(new FixedValueParameter<DoubleValue>("Beta", "The weighting factor beta", new DoubleValue(1.0)));
-      Parameters.Add(new ValueParameter<ITabularQualityFunction>("Quality function", "The quality function to use", new TabularAvgQualityFunction()));
+      Parameters.Add(new ValueParameter<ITabularStateValueFunction>("Quality function", "The quality function to use", new TabularAvgStateValueFunction()));
     }
 
-    protected sealed override int Select(object state, IEnumerable<int> actions, IRandom random) {
-
+    protected sealed override int Select(IReadOnlyList<object> followStates, IRandom random) {
+      var idxs = Enumerable.Range(0, followStates.Count);
       // find best action
-      var bestActions = new List<int>();
       var bestQuality = double.NegativeInfinity;
-      if (actions.Any(a => QualityFunction.Tries(state, a) == 0)) {
-        return actions.Where(a => QualityFunction.Tries(state, a) == 0).SampleRandom(random, 1).First();
+      if (followStates.Any(s => StateValueFunction.Tries(s) == 0)) {
+        return idxs.Where(idx => StateValueFunction.Tries(followStates[idx]) == 0).SampleRandom(random);
       }
 
       // windowing
-      var max = actions.Select(a => QualityFunction.Q(state, a)).Max();
-      var min = actions.Select(a => QualityFunction.Q(state, a)).Min();
+      var max = followStates.Select(s => StateValueFunction.Value(s)).Max();
+      var min = followStates.Select(s => StateValueFunction.Value(s)).Min();
       double range = max - min;
-      if (range.IsAlmost(0.0)) return actions.SampleRandom(random, 1).First();
+      if (range.IsAlmost(0.0)) return idxs.SampleRandom(random);
 
-      var w = from a in actions
-              select Math.Exp(Beta * (QualityFunction.Q(state, a) - min) / range);
+      var w = from s in followStates
+              select Math.Exp(Beta * (StateValueFunction.Value(s) - min) / range);
 
-      return actions.SampleProportional(random, 1, w).First();
+      return idxs.SampleProportional(random, 1, w).First();
 
     }
 
-    public sealed override void Update(IEnumerable<Tuple<object, int>> stateActionSequence, double quality) {
-      foreach (var t in stateActionSequence) {
-        var state = t.Item1;
-        var action = t.Item2;
-        QualityFunction.Update(state, action, quality);
+    public sealed override void Update(IEnumerable<object> stateSequence, double quality) {
+      foreach (var state in stateSequence) {
+        StateValueFunction.Update(state, quality);
       }
     }
 
-    protected override object CreateState(ISymbolicExpressionTreeNode root, List<int> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
-      return QualityFunction.StateFunction.CreateState(root, actions, parent, childIdx);
+    protected override object CreateState(ISymbolicExpressionTreeNode root, List<ISymbol> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
+      return StateValueFunction.StateFunction.CreateState(root, actions, parent, childIdx);
     }
 

branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies/EpsGreedySymbolicExpressionConstructionPolicy.cs

@@ -20 +20 @@
     }
 
-    public IQualityFunction QualityFunction {
+    public IStateValueFunction StateValueFunction {
       get {
-        return ((IValueParameter<IQualityFunction>)Parameters["Quality function"]).Value;
+        return ((IValueParameter<IStateValueFunction>)Parameters["Quality function"]).Value;
       }
-      set { ((IValueParameter<IQualityFunction>)Parameters["Quality function"]).Value = value; }
+      set { ((IValueParameter<IStateValueFunction>)Parameters["Quality function"]).Value = value; }
     }
 
@@ -30 +30 @@
       : base() {
       Parameters.Add(new FixedValueParameter<DoubleValue>("Eps", "The fraction of random pulls", new PercentValue(0.1, true)));
-      Parameters.Add(new ValueParameter<IQualityFunction>("Quality function", "The quality function to use", new TabularAvgQualityFunction()));
+      Parameters.Add(new ValueParameter<IStateValueFunction>("Quality function", "The quality function to use", new TabularAvgStateValueFunction()));
     }
 
-    protected override int Select(object state, IEnumerable<int> actions, IRandom random) {
+    protected override int Select(IReadOnlyList<object> followStates, IRandom random) {
+      var idxs = Enumerable.Range(0, followStates.Count);
       if (random.NextDouble() < Eps) {
-        return actions.SampleRandom(random, 1).First();
+        return idxs.SampleRandom(random);
       }
 
       // find best action
-      var bestActions = new List<int>();
+      var bestFollowStates = new List<int>();
       var bestQuality = double.NegativeInfinity;
-      foreach (var a in actions) {
-        double quality = QualityFunction.Q(state, a);
+      for (int idx = 0; idx < followStates.Count; idx++) {
+        double quality = StateValueFunction.Value(followStates[idx]);
 
         if (quality >= bestQuality) {
           if (quality > bestQuality) {
-            bestActions.Clear();
+            bestFollowStates.Clear();
             bestQuality = quality;
           }
-          bestActions.Add(a);
+          bestFollowStates.Add(idx);
         }
       }
-      return bestActions.SampleRandom(random, 1).First();
+      return bestFollowStates.SampleRandom(random);
     }
 
-    public override void Update(IEnumerable<Tuple<object, int>> stateActionSequence, double quality) {
-      foreach (var t in stateActionSequence) {
-        var state = t.Item1;
-        var action = t.Item2;
-        QualityFunction.Update(state, action, quality);
+    public sealed override void Update(IEnumerable<object> stateSequence, double quality) {
+      foreach (var state in stateSequence) {
+        StateValueFunction.Update(state, quality);
       }
     }
 
-    protected override object CreateState(ISymbolicExpressionTreeNode root, List<int> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
-      return QualityFunction.StateFunction.CreateState(root, actions, parent, childIdx);
+    protected override object CreateState(ISymbolicExpressionTreeNode root, List<ISymbol> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
+      return StateValueFunction.StateFunction.CreateState(root, actions, parent, childIdx);
     }
 

branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies/RandomSymbolicExpressionConstructionPolicy.cs

@@ -17 +17 @@
     }
 
-    protected override int Select(object state, IEnumerable<int> actions, IRandom random) {
-      return actions.SampleRandom(random, 1).First();
+    protected override int Select(IReadOnlyList<object> followStates, IRandom random) {
+      var idxs = Enumerable.Range(0, followStates.Count);
+      return idxs.SampleRandom(random);
     }
 
-    public override void Update(IEnumerable<Tuple<object, int>> stateActionSequence, double quality) {
+    public sealed override void Update(IEnumerable<object> stateSequence, double quality) {
+
       // ignore
     }
 
-    protected override object CreateState(ISymbolicExpressionTreeNode root, List<int> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
+    protected override object CreateState(ISymbolicExpressionTreeNode root, List<ISymbol> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
       return null; // doesn't use state information
     }

branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies/SymbolicExpressionConstructionPolicyBase.cs

@@ -21 +21 @@
     [Storable]
     public IRandom Random { get; private set; }
-    private SearchTree searchTree;
+    private SearchTree<ISymbol> searchTree; // tree of replacement symbols
 
     private class Slot {
@@ -31 +31 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
-      searchTree = new SearchTree();
+      searchTree = new SearchTree<ISymbol>();
     }
     protected SymbolicExpressionConstructionPolicyBase(SymbolicExpressionConstructionPolicyBase original, Cloner cloner)
@@ -39 +39 @@
 
       // search tree is not cloned or stored
-      searchTree = new SearchTree();
+      searchTree = new SearchTree<ISymbol>();
     }
 
@@ -52 +52 @@
       this.Problem = problem;
       this.Random = random;
-      this.searchTree = new SearchTree(); // represents all realized actionSequences as a prefix tree 
+      this.searchTree = new SearchTree<ISymbol>(); // represents all realized actionSequences as a prefix tree 
     }
 
-    public ISymbolicExpressionTree Sample(out IEnumerable<Tuple<object, int>> stateActionSequence) {
-      var actions = new List<int>();
+    public ISymbolicExpressionTree Sample(out IEnumerable<object> stateSequence) {
+      var actions = new List<ISymbol>();
       var states = new List<object>();
 
@@ -67 +67 @@
 
       Contract.Assert(Problem.Encoding.FunctionDefinitions == 0);
-      openSlots.Push(new Slot() { parent = root, childIdx = 0, minSize = 2 }); // at least two nodes are necessary below root
+      openSlots.Push(new Slot() { parent = root, childIdx = 0, minSize = g.GetMinimumExpressionLength(root.Symbol) - 1 }); // at least two nodes are necessary below root
 
       // tree size lower bound is the current tree size + the sum of the minimal size for all open slots
@@ -79 +79 @@
         var childIdx = next.childIdx;
 
-        // states might be defined differently be different policies
-        // this allows policies to calculate the state as a function of the current tree and the position where it is changed,
-        // or as a function of the list of actions so far, 
-        // or as a function of both
-        var currentState = CreateState(root, actions, parent, childIdx);
-        states.Add(currentState);
+        if (searchTree.IsLeafNode()) {
+          var allowedChildSymbols = g.GetAllowedChildSymbols(parent.Symbol, childIdx)
+            .Where(a => a.Enabled)
+            .Where(a => treeSize + g.GetMinimumExpressionLength(a) + openSlots.Select(e => e.minSize).Sum() <= maxLen);
 
-
-        // TODO: only filter the first time later use info from search tree
-        var alts = g.GetAllowedChildSymbols(parent.Symbol, childIdx)
-          .Where(a => treeSize + g.GetMinimumExpressionLength(a) + openSlots.Select(e => e.minSize).Sum() <= maxLen)
-          .ToArray();
-
-        if (searchTree.IsLeafNode()) {
-          searchTree.ExpandCurrentNode(alts);
+          searchTree.ExpandCurrentNode(allowedChildSymbols);
         }
 
-        if (!searchTree.PossibleActions.Any()) {
+        if (!searchTree.ChildValues.Any()) {
           throw new InvalidProgramException(string.Format("Couldn't construct a valid tree of maximum length {0} or all possible trees have been visited", maxLen));
         }
 
-        // select a symbol randomly for the child 
-        // select random alternative
-        var selectedIdx = Select(currentState, searchTree.PossibleActions, Random);
-        actions.Add(selectedIdx);
+        var alternatives = searchTree.ChildValues.ToArray(); // TODO perf
 
-        // and add child node to parent
-        var childNode = alts[selectedIdx].CreateTreeNode();
-        if (childNode.HasLocalParameters) {
-          throw new NotSupportedException("Symbols with parameters are not supported by construction policies for symbolic expressions. Try to reformulate the problem so that only discrete actions are necessary");
-          // childNode.ResetLocalParameters(Random);
+        // generate follow states
+        var followStates = new object[alternatives.Length];
+        for (int i = 0; i < followStates.Length; i++) {
+          // temporarily make the replacement and create the followState object
+          var childNode = alternatives[i].CreateTreeNode();
+          if (childNode.HasLocalParameters) {
+            throw new NotSupportedException("Symbols with parameters are not supported by construction policies for symbolic expressions. " +
+                                            "Try to reformulate the problem so that only discrete actions are necessary");
+            // childNode.ResetLocalParameters(Random);
+          }
+          parent.AddSubtree(childNode);
+          actions.Add(alternatives[i]);
+
+          // states might be defined differently be different policies
+          // this allows policies to calculate the state as a function of the current tree and the position where it is changed,
+          // or as a function of the list of actions so far, 
+          // or as a function of both
+          followStates[i] = CreateState(root, actions, parent, childIdx);
+
+          // roll back the change
+          parent.RemoveSubtree(parent.SubtreeCount - 1);
+          actions.RemoveAt(actions.Count - 1);
         }
 
-        Contract.Assert(parent.SubtreeCount == childIdx);
-        parent.AddSubtree(childNode); // enforce left-canonical derivation
-        treeSize++;
+        // select one of the follow states and prepare for the next step
+        var selectedIdx = Select(followStates, Random);
+        actions.Add(alternatives[selectedIdx]);
+        states.Add(followStates[selectedIdx]);
 
-        // push new slots 
-        for (int chIdx = g.GetMinimumSubtreeCount(childNode.Symbol) - 1; chIdx >= 0; chIdx--) {
-          int minForChild = g.GetAllowedChildSymbols(childNode.Symbol, chIdx).Min(a => g.GetMinimumExpressionLength(a)); // min length of all possible alts for the slot
-          openSlots.Push(new Slot() { parent = childNode, childIdx = chIdx, minSize = minForChild });
+        {
+          // and add child node to parent
+          var childNode = alternatives[selectedIdx].CreateTreeNode();
+
+          Contract.Assert(parent.SubtreeCount == childIdx); // enforce left-canonical derivation
+          parent.AddSubtree(childNode);
+          treeSize++;
+
+          // push new slots 
+          for (int chIdx = g.GetMinimumSubtreeCount(childNode.Symbol) - 1; chIdx >= 0; chIdx--) {
+            int minForChild = g.GetAllowedChildSymbols(childNode.Symbol, chIdx)
+              .Min(a => g.GetMinimumExpressionLength(a)); // min length of all possible alts for the slot
+            openSlots.Push(new Slot() { parent = childNode, childIdx = chIdx, minSize = minForChild });
+          }
         }
 
         // if this is the last slot we never have to revisit selectedIdx
         if (!openSlots.Any()) {
-          searchTree.RemoveBranch(selectedIdx);
+          searchTree.RemoveBranch(alternatives[selectedIdx]);
         } else {
-          searchTree.Follow(selectedIdx);
+          searchTree.Follow(alternatives[selectedIdx]);
         }
       }
 
-      stateActionSequence = states.Zip(actions, Tuple.Create);
+      stateSequence = states;
       return new SymbolicExpressionTree(root);
     }
 
+    /// <summary>
+    /// Choose one of the follow states
+    /// </summary>
+    /// <param name="followStates"></param>
+    /// <param name="random"></param>
+    /// <returns>The index of the selected follow state</returns>
+    protected abstract int Select(IReadOnlyList<object> followStates, IRandom random);
+    public abstract void Update(IEnumerable<object> stateSequence, double quality);
 
-    protected abstract int Select(object state, IEnumerable<int> possibleActions, IRandom random);
-    public abstract void Update(IEnumerable<Tuple<object, int>> stateActionSequence, double quality);
-
-    protected abstract object CreateState(ISymbolicExpressionTreeNode root, List<int> actions, ISymbolicExpressionTreeNode parent, int childIdx);
+    protected abstract object CreateState(ISymbolicExpressionTreeNode root, List<ISymbol> actions, ISymbolicExpressionTreeNode parent, int childIdx);
   }
 }

branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies/UcbSymbolicExpressionConstructionPolicy.cs

@@ -20 +20 @@
     }
 
-    public ITabularQualityFunction QualityFunction {
+    public ITabularStateValueFunction StateValueFunction {
       get {
-        return ((IValueParameter<ITabularQualityFunction>)Parameters["Quality function"]).Value;
+        return ((IValueParameter<ITabularStateValueFunction>)Parameters["Quality function"]).Value;
       }
-      set { ((IValueParameter<ITabularQualityFunction>)Parameters["Quality function"]).Value = value; }
+      set { ((IValueParameter<ITabularStateValueFunction>)Parameters["Quality function"]).Value = value; }
     }
 
@@ -39 +39 @@
       : base() {
       Parameters.Add(new FixedValueParameter<DoubleValue>("R", "The weighting factor for the confidence bound (should be scaled based on the range or the fitness values)", new DoubleValue(1.0)));
-      Parameters.Add(new ValueParameter<ITabularQualityFunction>("Quality function", "The quality function to use", new TabularAvgQualityFunction()));
+      Parameters.Add(new ValueParameter<ITabularStateValueFunction>("Quality function", "The quality function to use", new TabularAvgStateValueFunction()));
     }
 
-    protected sealed override int Select(object state, IEnumerable<int> actions, IRandom random) {
-
-      // find best action
-      var bestActions = new List<int>();
+    protected sealed override int Select(IReadOnlyList<object> followStates, IRandom random) {
+      var bestFollowStates = new List<int>();
       var bestQuality = double.NegativeInfinity;
-      int totalTries = actions.Sum(a => QualityFunction.Tries(state, a));
-      foreach (var a in actions) {
+      int totalTries = followStates.Sum(s => StateValueFunction.Tries(s));
+      for (int idx = 0; idx < followStates.Count; idx++) {
         double quality;
-        if (QualityFunction.Tries(state, a) == 0) {
+        var s = followStates[idx];
+        if (StateValueFunction.Tries(s) == 0) {
           quality = double.PositiveInfinity;
         } else {
-          quality = QualityFunction.Q(state, a) + R * Math.Sqrt((2 * Math.Log(totalTries)) / QualityFunction.Tries(state, a));
+          quality = StateValueFunction.Value(s) + R * Math.Sqrt((2 * Math.Log(totalTries)) / StateValueFunction.Tries(s));
         }
         if (quality >= bestQuality) {
           if (quality > bestQuality) {
-            bestActions.Clear();
+            bestFollowStates.Clear();
             bestQuality = quality;
           }
-          bestActions.Add(a);
+          bestFollowStates.Add(idx);
         }
       }
-      return bestActions.SampleRandom(random, 1).First();
+      return bestFollowStates.SampleRandom(random);
     }
 
-    public sealed override void Update(IEnumerable<Tuple<object, int>> stateActionSequence, double quality) {
-      foreach (var t in stateActionSequence) {
-        var state = t.Item1;
-        var action = t.Item2;
-        QualityFunction.Update(state, action, quality);
+    public sealed override void Update(IEnumerable<object> stateSequence, double quality) {
+      foreach (var state in stateSequence) {
+        StateValueFunction.Update(state, quality);
       }
     }
 
-    protected override object CreateState(ISymbolicExpressionTreeNode root, List<int> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
-      return QualityFunction.StateFunction.CreateState(root, actions, parent, childIdx);
+    protected override object CreateState(ISymbolicExpressionTreeNode root, List<ISymbol> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
+      return StateValueFunction.StateFunction.CreateState(root, actions, parent, childIdx);
     }
 

branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies/UcbTunedSymbolicExpressionConstructionPolicy.cs

@@ -20 +20 @@
     }
 
-    public ITabularQualityFunction QualityFunction {
+    public ITabularStateValueFunction StateValueFunction {
       get {
-        return ((IValueParameter<ITabularQualityFunction>)Parameters["Quality function"]).Value;
+        return ((IValueParameter<ITabularStateValueFunction>)Parameters["Quality function"]).Value;
       }
-      set { ((IValueParameter<ITabularQualityFunction>)Parameters["Quality function"]).Value = value; }
+      set { ((IValueParameter<ITabularStateValueFunction>)Parameters["Quality function"]).Value = value; }
     }
 
@@ -39 +39 @@
       : base() {
       Parameters.Add(new FixedValueParameter<DoubleValue>("R", "The weighting factor for the confidence bound (should be scaled based on the range or the fitness values)", new DoubleValue(1.0)));
-      Parameters.Add(new ValueParameter<ITabularQualityFunction>("Quality function", "The quality function to use", new TabularAvgQualityFunction()));
+      Parameters.Add(new ValueParameter<ITabularStateValueFunction>("Quality function", "The quality function to use", new TabularAvgStateValueFunction()));
     }
 
-    protected sealed override int Select(object state, IEnumerable<int> actions, IRandom random) {
-
-      // find best action
-      var bestActions = new List<int>();
+    protected sealed override int Select(IReadOnlyList<object> followStates, IRandom random) {
+      var bestFollowStates = new List<int>();
       var bestQuality = double.NegativeInfinity;
-      int totalTries = actions.Sum(a => QualityFunction.Tries(state, a));
-      foreach (var a in actions) {
+      int totalTries = followStates.Sum(s => StateValueFunction.Tries(s));
+      for (int idx = 0; idx < followStates.Count; idx++) {
+        var s = followStates[idx];
         double quality;
-        if (QualityFunction.Tries(state, a) == 0) {
+        if (StateValueFunction.Tries(s) == 0) {
           quality = double.PositiveInfinity;
         } else {
-          double v = QualityFunction.QVariance(state, a) + Math.Sqrt(2 * Math.Log(totalTries) / QualityFunction.Tries(state, a));
-          quality = QualityFunction.Q(state, a) + R * Math.Sqrt(Math.Log(totalTries) / QualityFunction.Tries(state, a) * v);
+          double v = StateValueFunction.ValueVariance(s) + Math.Sqrt(2 * Math.Log(totalTries) / StateValueFunction.Tries(s));
+          quality = StateValueFunction.Value(s) + R * Math.Sqrt(Math.Log(totalTries) / StateValueFunction.Tries(s) * v);
         }
         if (quality >= bestQuality) {
           if (quality > bestQuality) {
-            bestActions.Clear();
+            bestFollowStates.Clear();
             bestQuality = quality;
           }
-          bestActions.Add(a);
+          bestFollowStates.Add(idx);
         }
       }
-      return bestActions.SampleRandom(random, 1).First();
+      return bestFollowStates.SampleRandom(random);
     }
 
-    public sealed override void Update(IEnumerable<Tuple<object, int>> stateActionSequence, double quality) {
-      foreach (var t in stateActionSequence) {
-        var state = t.Item1;
-        var action = t.Item2;
-        QualityFunction.Update(state, action, quality);
+    public sealed override void Update(IEnumerable<object> stateSequence, double quality) {
+      foreach (var state in stateSequence) {
+        StateValueFunction.Update(state, quality);
       }
     }
 
-    protected override object CreateState(ISymbolicExpressionTreeNode root, List<int> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
-      return QualityFunction.StateFunction.CreateState(root, actions, parent, childIdx);
+    protected override object CreateState(ISymbolicExpressionTreeNode root, List<ISymbol> actions, ISymbolicExpressionTreeNode parent, int childIdx) {
+      return StateValueFunction.StateFunction.CreateState(root, actions, parent, childIdx);
     }