Changeset 11770 for branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/GrammarPolicies/RandomNoResamplingPolicy.cs

Timestamp:

01/15/15 18:59:07 (9 years ago)

Author:

gkronber

Message:

#2283: worked on generic sequential search alg with bandit policy as parameter

File:

• branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/GrammarPolicies/RandomNoResamplingPolicy.cs (modified) (1 diff)

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/GrammarPolicies/RandomNoResamplingPolicy.cs

@@ -1 +1 @@
 using System;
 using System.Collections.Generic;
+using System.Configuration;
 using System.Linq;
+using System.Security.Policy;
 using System.Text;
-using System.Threading.Tasks;
 using HeuristicLab.Common;
 using HeuristicLab.Problems.GrammaticalOptimization;
 
 namespace HeuristicLab.Algorithms.Bandits.GrammarPolicies {
-  public class RandomNoResamplingPolicy : IGrammarPolicy {
+  public class RandomNoResamplingPolicy : GrammarPolicy {
 
-    private readonly Dictionary<ReadonlySequence, bool> done;
-    private readonly Dictionary<Tuple<ReadonlySequence, ReadonlySequence>, ReadonlySequence> nextState;
+    private readonly HashSet<string> done;
 
-
-    public RandomNoResamplingPolicy() {
-      this.done = new Dictionary<ReadonlySequence, bool>();
+    public RandomNoResamplingPolicy(IProblem problem, bool useCanonicalRepresentation)
+      : base(problem, useCanonicalRepresentation) {
+      this.done = new HashSet<string>();
     }
 
-    public ReadonlySequence SelectAction(Random random, ReadonlySequence state, IEnumerable<ReadonlySequence> actions) {
-      var allDone = true;
-      foreach (var a in actions) {
-        var p = Tuple.Create(state, a);
-        allDone &= nextState.ContainsKey(p) && Done(nextState[p]);
-        if (!allDone) break;
+    public override bool TrySelect(Random random, ReadonlySequence curState, IEnumerable<ReadonlySequence> afterStates, out ReadonlySequence selectedState) {
+      // only select states that are not yet done
+      afterStates = afterStates.Where(a => !done.Contains(a.ToString())).ToArray();
+      if (!afterStates.Any()) {
+        // fail because all follow states have already been visited => also disable the current state
+        done.Add(CanonicalState(curState.ToString()));
+        selectedState = null;
+        return false;
       }
-      if(allDone)
-      return actions
-        .Where(a => !nextState.ContainsKey(Tuple.Create(state, a)) || Done(nextState[Tuple.Create(state, a)]))
-        .SelectRandom(random);
+
+      selectedState = afterStates.SelectRandom(random);
+      return true;
     }
 
-    public void UpdateReward(ReadonlySequence state, ReadonlySequence action, double reward, ReadonlySequence newState) {
-      var key = Tuple.Create(state, action);
-      nextState[key] = newState;
-      if (newState.IsTerminal) done[newState] = true;
-      if 
+    public override void UpdateReward(IEnumerable<ReadonlySequence> stateTrajectory, double reward) {
+      base.UpdateReward(stateTrajectory, reward);
+      // ignore rewards but update the set of visited terminal states
+
+      // the last state could be terminal
+      var lastState = stateTrajectory.Last();
+      if (lastState.IsTerminal) done.Add(CanonicalState(lastState.ToString()));
     }
 
-    public bool Done(ReadonlySequence state) {
-      return done.ContainsKey(state);
+    public override void Reset() {
+      base.Reset();
+      done.Clear();
     }
   }