Changeset 11727 for branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs

Timestamp:

12/29/14 11:02:36 (9 years ago)

Author:

gkronber

Message:

#2283: worked on grammatical optimization problem solvers (simple MCTS done)

File:

• branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs (modified) (2 diffs)

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs

@@ -1 +1 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Text;
@@ -11 +12 @@
     private readonly int[] tries;
     private readonly double[] sumReward;
+    private readonly RandomPolicy randomPolicy;
+
     public EpsGreedyPolicy(Random random, int numActions, double eps)
       : base(numActions) {
       this.random = random;
       this.eps = eps;
-      this.tries = new int[NumActions];
-      this.sumReward = new double[NumActions];
+      this.randomPolicy = new RandomPolicy(random, numActions);
+      this.tries = new int[numActions];
+      this.sumReward = new double[numActions];
     }
 
     public override int SelectAction() {
+      Debug.Assert(Actions.Any());
       if (random.NextDouble() > eps) {
         // select best
         var maxReward = double.NegativeInfinity;
         int bestAction = -1;
-        for (int i = 0; i < NumActions; i++) {
-          if (tries[i] == 0) return i;
-          var avgReward = sumReward[i] / tries[i];
+        foreach (var a in Actions) {
+          if (tries[a] == 0) return a;
+          var avgReward = sumReward[a] / tries[a];
           if (maxReward < avgReward) {
             maxReward = avgReward;
-            bestAction = i;
+            bestAction = a;
           }
         }
+        Debug.Assert(bestAction >= 0);
         return bestAction;
       } else {
         // select random
-        return random.Next(NumActions);
+        return randomPolicy.SelectAction();
       }
     }
     public override void UpdateReward(int action, double reward) {
+      Debug.Assert(Actions.Contains(action));
+
+      randomPolicy.UpdateReward(action, reward); // does nothing
       tries[action]++;
       sumReward[action] += reward;
     }
+
+    public override void DisableAction(int action) {
+      base.DisableAction(action);
+      randomPolicy.DisableAction(action);
+      sumReward[action] = 0;
+      tries[action] = -1;
+    }
+
     public override void Reset() {
+      base.Reset();
+      randomPolicy.Reset();
       Array.Clear(tries, 0, tries.Length);
       Array.Clear(sumReward, 0, sumReward.Length);