source: branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.Bandits/Models/GaussianModel.cs @ 11730

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using HeuristicLab.Common;

namespace HeuristicLab.Algorithms.Bandits.Models {
  // bayesian estimation of a Gaussian with unknown mean and known variance
  public class GaussianModel : IModel {
    private readonly int numActions;
    private readonly int[] tries;
    private readonly double[] sumRewards;


    // parameters of Gaussian prior for mean
    private readonly double meanPriorMu;
    private readonly double meanPriorVariance;

    private readonly double rewardVariance = 0.1; // assumed know reward variance

    public GaussianModel(int numActions, double meanPriorMu, double meanPriorVariance) {
      this.numActions = numActions;
      this.tries = new int[numActions];
      this.sumRewards = new double[numActions];
      this.meanPriorMu = meanPriorMu;
      this.meanPriorVariance = meanPriorVariance;
    }


    public double[] SampleExpectedRewards(Random random) {
      // expected values for reward
      var theta = new double[numActions];

      for (int a = 0; a < numActions; a++) {
        if (tries[a] == -1) {
          theta[a] = double.NegativeInfinity; // disabled action
        } else {
          // calculate posterior mean and variance (for mean reward)

          // see Murphy 2007: Conjugate Bayesian analysis of the Gaussian distribution (http://www.cs.ubc.ca/~murphyk/Papers/bayesGauss.pdf)
          var posteriorVariance = 1.0 / (tries[a] / rewardVariance + 1.0 / meanPriorVariance);
          var posteriorMean = posteriorVariance * (meanPriorMu / meanPriorVariance + sumRewards[a] / rewardVariance);

          // sample a mean from the posterior 
          theta[a] = Rand.RandNormal(random) * Math.Sqrt(posteriorVariance) + posteriorMean;
          // theta already represents the expected reward value => nothing else to do
        }
      }

      return theta;
    }

    public void Update(int action, double reward) {
      sumRewards[action] += reward;
      tries[action]++;
    }

    public void Disable(int action) {
      tries[action] = -1;
      sumRewards[action] = 0.0;
    }

    public void Reset() {
      Array.Clear(tries, 0, numActions);
      Array.Clear(sumRewards, 0, numActions);
    }

    public void PrintStats() {
      for (int i = 0; i < numActions; i++) {
        Console.Write("{0:F2} ", sumRewards[i] / (double)tries[i]);
      }
    }
  }
}