source: branches/HeuristicLab.Algorithms.IteratedSentenceConstruction/HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction/3.3/Policies/EpsGreedySymbolicExpressionConstructionPolicy.cs @ 12955

using System;
using System.Collections.Generic;
using System.Linq;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
using HeuristicLab.Parameters;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Random;

namespace HeuristicLab.Algorithms.IteratedSymbolicExpressionConstruction {
  [StorableClass]
  [Item("EpsGreedySymbolicExpressionConstructionPolicy", "")]
  public class EpsGreedySymbolicExpressionConstructionPolicy : SymbolicExpressionConstructionPolicyBase {

    public double Eps {
      get { return ((IFixedValueParameter<DoubleValue>)Parameters["Eps"]).Value.Value; }
      set { ((IFixedValueParameter<DoubleValue>)Parameters["Eps"]).Value.Value = value; }
    }

    public IStateValueFunction StateValueFunction {
      get {
        return ((IValueParameter<IStateValueFunction>)Parameters["Quality function"]).Value;
      }
      set { ((IValueParameter<IStateValueFunction>)Parameters["Quality function"]).Value = value; }
    }

    public EpsGreedySymbolicExpressionConstructionPolicy()
      : base() {
      Parameters.Add(new FixedValueParameter<DoubleValue>("Eps", "The fraction of random pulls", new PercentValue(0.1, true)));
      Parameters.Add(new ValueParameter<IStateValueFunction>("Quality function", "The quality function to use", new TabularAvgStateValueFunction()));
    }

    protected override int Select(IReadOnlyList<object> followStates, IRandom random) {
      var idxs = Enumerable.Range(0, followStates.Count);
      if (random.NextDouble() < Eps) {
        return idxs.SampleRandom(random);
      }

      // find best action
      var bestFollowStates = new List<int>();
      var bestQuality = double.NegativeInfinity;
      for (int idx = 0; idx < followStates.Count; idx++) {
        double quality = StateValueFunction.Value(followStates[idx]);

        if (quality >= bestQuality) {
          if (quality > bestQuality) {
            bestFollowStates.Clear();
            bestQuality = quality;
          }
          bestFollowStates.Add(idx);
        }
      }
      return bestFollowStates.SampleRandom(random);
    }

    public sealed override void Update(IEnumerable<object> stateSequence, double quality) {
      foreach (var state in stateSequence) {
        StateValueFunction.Update(state, quality);
      }
    }

    protected override object CreateState(ISymbolicExpressionTreeNode root, List<ISymbol> actionSequence, ISymbolicExpressionTreeNode parent, int childIdx) {
      return StateValueFunction.StateFunction.CreateState(root, actionSequence, parent, childIdx);
    }

    #region IItem
    protected EpsGreedySymbolicExpressionConstructionPolicy(EpsGreedySymbolicExpressionConstructionPolicy original, Cloner cloner)
      : base(original, cloner) {
    }

    [StorableConstructor]
    protected EpsGreedySymbolicExpressionConstructionPolicy(bool deserializing) : base(deserializing) { }

    public override HeuristicLab.Common.IDeepCloneable Clone(HeuristicLab.Common.Cloner cloner) {
      return new EpsGreedySymbolicExpressionConstructionPolicy(this, cloner);
    }

    #endregion
  }
}