source: branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/UCBNormalPolicy.cs @ 11732

Last change on this file since 11732 was 11732, checked in by gkronber, 6 years ago

#2283: refactoring and bug fixes

File size: 1.6 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Diagnostics;
4using System.Linq;
5using System.Text;
6using System.Threading.Tasks;
7
8namespace HeuristicLab.Algorithms.Bandits {
9  public class UCBNormalPolicy : IPolicy {
10
11    public int SelectAction(Random random, IEnumerable<IPolicyActionInfo> actionInfos) {
12      var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>().ToArray(); // TODO: performance
13      int bestAction = -1;
14      double bestQ = double.NegativeInfinity;
15      int totalTries = myActionInfos.Where(a => !a.Disabled).Sum(a => a.Tries);
16
17      for (int a = 0; a < myActionInfos.Length; a++) {
18        if (myActionInfos[a].Disabled) continue;
19        if (totalTries <= 1 || myActionInfos[a].Tries <= 1 || myActionInfos[a].Tries <= Math.Ceiling(8 * Math.Log(totalTries))) return a;
20
21        var tries = myActionInfos[a].Tries;
22        var avgReward = myActionInfos[a].AvgReward;
23        var rewardVariance = myActionInfos[a].RewardVariance;
24        var estVariance = 16 * rewardVariance * (Math.Log(totalTries - 1) / tries);
25        if (estVariance < 0) estVariance = 0; // numerical problems
26        var q = avgReward
27          + Math.Sqrt(estVariance);
28        if (q > bestQ) {
29          bestQ = q;
30          bestAction = a;
31        }
32      }
33      Debug.Assert(bestAction > -1);
34      return bestAction;
35    }
36
37    public IPolicyActionInfo CreateActionInfo() {
38      return new MeanAndVariancePolicyActionInfo();
39    }
40
41    public override string ToString() {
42      return "UCBNormalPolicy";
43    }
44  }
45}
Note: See TracBrowser for help on using the repository browser.