Changeset 11730 for branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/UCBNormalPolicy.cs

Timestamp:

01/02/15 16:08:21 (10 years ago)

Author:

gkronber

Message:

#2283: several major extensions for grammatical optimization

File:

• branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/UCBNormalPolicy.cs (modified) (3 diffs)

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/UCBNormalPolicy.cs

@@ -24 +24 @@
       double bestQ = double.NegativeInfinity;
       foreach (var a in Actions) {
-        if (totalTries == 0 || tries[a] == 0 || tries[a] < Math.Ceiling(8 * Math.Log(totalTries))) return a;
+        if (totalTries <= 1 || tries[a] <= 1 || tries[a] <= Math.Ceiling(8 * Math.Log(totalTries))) return a;
         var avgReward = sumReward[a] / tries[a];
+        var estVariance = 16 * ((sumSqrReward[a] - tries[a] * Math.Pow(avgReward, 2)) / (tries[a] - 1)) * (Math.Log(totalTries - 1) / tries[a]);
+        if (estVariance < 0) estVariance = 0; // numerical problems
         var q = avgReward
-          + Math.Sqrt(16 * ((sumSqrReward[a] - tries[a] * Math.Pow(avgReward, 2)) / (tries[a] - 1)) * (Math.Log(totalTries - 1) / tries[a]));
+          + Math.Sqrt(estVariance);
         if (q > bestQ) {
           bestQ = q;
@@ -33 +35 @@
         }
       }
+      Debug.Assert(Actions.Contains(bestAction));
       return bestAction;
     }
@@ -58 +61 @@
       Array.Clear(sumSqrReward, 0, sumSqrReward.Length);
     }
+    public override void PrintStats() {
+      for (int i = 0; i < sumReward.Length; i++) {
+        if (tries[i] >= 0) {
+          Console.Write("{0,5:F2}", sumReward[i] / tries[i]);
+        } else {
+          Console.Write("{0,5}", "");
+        }
+      }
+      Console.WriteLine();
+    }
+    public override string ToString() {
+      return "UCBNormalPolicy";
+    }
   }
 }