source: branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/BanditPolicies/UCBNormalPolicy.cs @ 11742

Last change on this file since 11742 was 11742, checked in by gkronber, 6 years ago

#2283 refactoring

File size: 1.5 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Diagnostics;
4using System.Linq;
5using System.Text;
6using System.Threading.Tasks;
7
8namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
9  public class UCBNormalPolicy : IBanditPolicy {
10
11    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
12      var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
13      int bestAction = -1;
14      double bestQ = double.NegativeInfinity;
15      int totalTries = myActionInfos.Where(a => !a.Disabled).Sum(a => a.Tries);
16      int aIdx = -1;
17      foreach (var aInfo in myActionInfos) {
18        aIdx++;
19        if (aInfo.Disabled) continue;
20        if (totalTries <= 1 || aInfo.Tries <= 1 || aInfo.Tries <= Math.Ceiling(8 * Math.Log(totalTries))) return aIdx;
21
22        var tries = aInfo.Tries;
23        var avgReward = aInfo.AvgReward;
24        var rewardVariance = aInfo.RewardVariance;
25        var estVariance = 16.0 * rewardVariance * (Math.Log(totalTries - 1) / tries);
26        var q = avgReward + Math.Sqrt(estVariance);
27        if (q > bestQ) {
28          bestQ = q;
29          bestAction = aIdx;
30        }
31      }
32      Debug.Assert(bestAction > -1);
33      return bestAction;
34    }
35
36    public IBanditPolicyActionInfo CreateActionInfo() {
37      return new MeanAndVariancePolicyActionInfo();
38    }
39
40    public override string ToString() {
41      return "UCBNormalPolicy";
42    }
43  }
44}
Note: See TracBrowser for help on using the repository browser.