source: branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs @ 11732

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace HeuristicLab.Algorithms.Bandits {
  public class EpsGreedyPolicy : IPolicy {
    private readonly double eps;
    private readonly RandomPolicy randomPolicy;

    public EpsGreedyPolicy(double eps) {
      this.eps = eps;
      this.randomPolicy = new RandomPolicy();
    }
    public int SelectAction(Random random, IEnumerable<IPolicyActionInfo> actionInfos) {
      Debug.Assert(actionInfos.Any());
      if (random.NextDouble() > eps) {
        // select best
        var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
        int bestAction = -1;
        double bestQ = double.NegativeInfinity;
        int aIdx = -1;
        foreach (var aInfo in myActionInfos) {

          aIdx++;
          if (aInfo.Disabled) continue;
          if (aInfo.Tries == 0) return aIdx;


          var avgReward = aInfo.SumReward / aInfo.Tries;         
          //var q = avgReward;
          var q = aInfo.MaxReward;
          if (q > bestQ) {
            bestQ = q;
            bestAction = aIdx;
          }
        }
        Debug.Assert(bestAction >= 0);
        return bestAction;
      } else {
        // select random
        return randomPolicy.SelectAction(random, actionInfos);
      }
    }

    public IPolicyActionInfo CreateActionInfo() {
      return new DefaultPolicyActionInfo();
    }


    public override string ToString() {
      return string.Format("EpsGreedyPolicy({0:F2})", eps);
    }
  }
}