Changeset 11792 for branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/BanditPolicies/UCB1TunedPolicy.cs

Timestamp:

01/16/15 18:26:35 (10 years ago)

Author:

gkronber

Message:

#2283 work-in-progress commit (does not compile)

File:

• branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/BanditPolicies/UCB1TunedPolicy.cs (modified) (2 diffs)

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/BanditPolicies/UCB1TunedPolicy.cs

@@ -5 +5 @@
 using System.Text;
 using System.Threading.Tasks;
+using HeuristicLab.Common;
 
 namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
@@ -12 +13 @@
     public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
       var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
-      int bestAction = -1;
-      double bestQ = double.NegativeInfinity;
+
       int totalTries = myActionInfos.Where(a => !a.Disabled).Sum(a => a.Tries);
 
       int aIdx = -1;
+      double bestQ = double.NegativeInfinity;
+      var bestActions = new List<int>();
       foreach (var aInfo in myActionInfos) {
         aIdx++;
         if (aInfo.Disabled) continue;
-        if (aInfo.Tries == 0) return aIdx;
+        double q;
+        if (aInfo.Tries == 0) {
+          q = double.PositiveInfinity;
+        } else {
+          var sumReward = aInfo.SumReward;
+          var tries = aInfo.Tries;
 
-        var sumReward = aInfo.SumReward;
-        var tries = aInfo.Tries;
-
-        var avgReward = sumReward / tries;
-        var q = avgReward + Math.Sqrt((Math.Log(totalTries) / tries) * Math.Min(1.0 / 4, V(aInfo, totalTries))); // 1/4 is upper bound of bernoulli distributed variable
+          var avgReward = sumReward / tries;
+          q = avgReward + Math.Sqrt((Math.Log(totalTries) / tries) * Math.Min(1.0 / 4, V(aInfo, totalTries)));
+          // 1/4 is upper bound of bernoulli distributed variable
+        }
         if (q > bestQ) {
           bestQ = q;
-          bestAction = aIdx;
+          bestActions.Clear();
+          bestActions.Add(aIdx);
+        } else if (q == bestQ) {
+          bestActions.Add(aIdx);
         }
       }
-      Debug.Assert(bestAction > -1);
-      return bestAction;
+      Debug.Assert(bestActions.Any());
+
+      return bestActions.SelectRandom(random);
     }