source: branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization.Test/TestBanditPolicies.cs @ 11727

using System;
using System.Linq;
using System.Collections.Generic;
using System.Globalization;
using HeuristicLab.Algorithms.Bandits;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
  [TestClass]
  public class TestBanditPolicies {
    [TestMethod]
    public void ComparePoliciesForBernoulliBandit() {
      System.Threading.Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
      var globalRand = new Random(31415);
      var seedForPolicy = globalRand.Next();
      var nArms = 10;
      //Console.WriteLine("Exp3 (gamma=0.01)");
      //TestPolicyBernoulli(globalRand, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 1));
      //Console.WriteLine("Exp3 (gamma=0.05)");
      //estPolicyBernoulli(globalRand, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 1));
      Console.WriteLine("Thompson (Bernoulli)");
      TestPolicyBernoulli(globalRand, nArms, new BernoulliThompsonSamplingPolicy(new Random(seedForPolicy), nArms));
      Console.WriteLine("Random");
      TestPolicyBernoulli(globalRand, nArms, new RandomPolicy(new Random(seedForPolicy), nArms));
      Console.WriteLine("UCB1");
      TestPolicyBernoulli(globalRand, nArms, new UCB1Policy(nArms));
      Console.WriteLine("UCB1Tuned");
      TestPolicyBernoulli(globalRand, nArms, new UCB1TunedPolicy(nArms));
      Console.WriteLine("UCB1Normal");
      TestPolicyBernoulli(globalRand, nArms, new UCBNormalPolicy(nArms));
      Console.WriteLine("Eps(0.01)");
      TestPolicyBernoulli(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.01));
      Console.WriteLine("Eps(0.05)");
      TestPolicyBernoulli(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.05));
      //Console.WriteLine("Eps(0.1)");
      //TestPolicyBernoulli(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.1));
      //Console.WriteLine("Eps(0.2)");
      //TestPolicyBernoulli(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.2));
      //Console.WriteLine("Eps(0.5)");
      //TestPolicyBernoulli(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.5));
    }
    [TestMethod]
    public void ComparePoliciesForNormalBandit() {
      System.Threading.Thread.CurrentThread.CurrentCulture = CultureInfo.InvariantCulture;
      var globalRand = new Random(31415);
      var seedForPolicy = globalRand.Next();
      var nArms = 10;
      Console.WriteLine("Thompson (Gaussian)");
      TestPolicyNormal(globalRand, nArms, new GaussianThompsonSamplingPolicy(new Random(seedForPolicy), nArms));
      Console.WriteLine("Random");
      TestPolicyNormal(globalRand, nArms, new RandomPolicy(new Random(seedForPolicy), nArms));
      Console.WriteLine("UCB1");
      TestPolicyNormal(globalRand, nArms, new UCB1Policy(nArms));
      Console.WriteLine("UCB1Tuned");
      TestPolicyNormal(globalRand, nArms, new UCB1TunedPolicy(nArms));
      Console.WriteLine("UCB1Normal");
      TestPolicyNormal(globalRand, nArms, new UCBNormalPolicy(nArms));
      //Console.WriteLine("Exp3 (gamma=0.01)");
      //TestPolicyNormal(globalRand, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 0.01));
      //Console.WriteLine("Exp3 (gamma=0.05)");
      //TestPolicyNormal(globalRand, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 0.05));
      Console.WriteLine("Eps(0.01)");
      TestPolicyNormal(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.01));
      Console.WriteLine("Eps(0.05)");
      TestPolicyNormal(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.05));
      //Console.WriteLine("Eps(0.1)");
      //TestPolicyNormal(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.1));
      //Console.WriteLine("Eps(0.2)");
      //TestPolicyNormal(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.2));
      //Console.WriteLine("Eps(0.5)");
      //TestPolicyNormal(globalRand, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.5));
    }

    private void TestPolicyBernoulli(Random globalRand, int nArms, IPolicy policy) {
      var maxIt = 1E6;
      var reps = 10; // 10 independent runs
      var avgRegretForIteration = new Dictionary<int, double>();
      // calculate statistics
      for (int r = 0; r < reps; r++) {
        var nextLogStep = 1;
        var b = new BernoulliBandit(new Random(globalRand.Next()), 10);
        policy.Reset();
        var totalRegret = 0.0;

        for (int i = 0; i <= maxIt; i++) {
          var selectedAction = policy.SelectAction();
          var reward = b.Pull(selectedAction);
          totalRegret += b.OptimalExpectedReward - reward;
          policy.UpdateReward(selectedAction, reward);
          if (i == nextLogStep) {
            nextLogStep *= 10;
            if (!avgRegretForIteration.ContainsKey(i)) {
              avgRegretForIteration.Add(i, 0.0);
            }
            avgRegretForIteration[i] += totalRegret / i;
          }
        }
      }
      // print
      foreach (var p in avgRegretForIteration.Keys.OrderBy(k => k)) {
        Console.WriteLine("{0} {1}", p, avgRegretForIteration[p] / reps); // print avg. of avg. regret
      }
    }
    private void TestPolicyNormal(Random globalRand, int nArms, IPolicy policy) {
      var maxIt = 1E6;
      var reps = 10; // 10 independent runs
      var avgRegretForIteration = new Dictionary<int, double>();
      // calculate statistics
      for (int r = 0; r < reps; r++) {
        var nextLogStep = 1;
        var b = new TruncatedNormalBandit(new Random(globalRand.Next()), 10);
        policy.Reset();
        var totalRegret = 0.0;

        for (int i = 0; i <= maxIt; i++) {
          var selectedAction = policy.SelectAction();
          var reward = b.Pull(selectedAction);
          totalRegret += b.OptimalExpectedReward - reward;
          policy.UpdateReward(selectedAction, reward);
          if (i == nextLogStep) {
            nextLogStep *= 10;
            if (!avgRegretForIteration.ContainsKey(i)) {
              avgRegretForIteration.Add(i, 0.0);
            }
            avgRegretForIteration[i] += totalRegret / i;
          }
        }
      }
      // print
      foreach (var p in avgRegretForIteration.Keys.OrderBy(k => k)) {
        Console.WriteLine("{0} {1}", p, avgRegretForIteration[p] / reps); // print avg. of avg. regret
      }
    }

  }
}