Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/BanditPolicies/UCBNormalPolicy.cs @ 11792

Last change on this file since 11792 was 11792, checked in by gkronber, 9 years ago

#2283 work-in-progress commit (does not compile)

File size: 1.7 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Diagnostics;
4using System.Linq;
5using System.Text;
6using System.Threading.Tasks;
7using HeuristicLab.Common;
8
9namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
10  public class UCBNormalPolicy : IBanditPolicy {
11
12    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
13      var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
14      int totalTries = myActionInfos.Where(a => !a.Disabled).Sum(a => a.Tries);
15      double bestQ = double.NegativeInfinity;
16      int aIdx = -1;
17      var bestActions = new List<int>();
18      foreach (var aInfo in myActionInfos) {
19        aIdx++;
20        if (aInfo.Disabled) continue;
21        double q;
22        if (totalTries <= 1 || aInfo.Tries <= 1 || aInfo.Tries <= Math.Ceiling(8 * Math.Log(totalTries))) {
23          q = double.PositiveInfinity;
24        } else {
25          var tries = aInfo.Tries;
26          var avgReward = aInfo.AvgReward;
27          var rewardVariance = aInfo.RewardVariance;
28          var estVariance = 16.0 * rewardVariance * (Math.Log(totalTries - 1) / tries);
29          q = avgReward + Math.Sqrt(estVariance);
30        }
31        if (q > bestQ) {
32          bestQ = q;
33          bestActions.Clear();
34          bestActions.Add(aIdx);
35        } else if (q == bestQ) {
36          bestActions.Add(aIdx);
37        }
38      }
39      Debug.Assert(bestActions.Any());
40      return bestActions.SelectRandom(random);
41    }
42
43    public IBanditPolicyActionInfo CreateActionInfo() {
44      return new MeanAndVariancePolicyActionInfo();
45    }
46
47    public override string ToString() {
48      return "UCBNormalPolicy";
49    }
50  }
51}
Note: See TracBrowser for help on using the repository browser.