Context Navigation

HeuristicLab.Algorithms.Bandits

Timestamp:

01/27/15 16:34:34 (10 years ago)

Author:

gkronber

Message:

linear value function approximation and good results for poly-10 benchmark

Location:

Files:

-                      r11747
+                      r11832
 namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
   public class BernoulliPolicyActionInfo : IBanditPolicyActionInfo {
-    private double knownValue;
-    public bool Disabled { get { return NumSuccess == -1; } }
     public int NumSuccess { get; private set; }
     public int NumFailure { get; private set; }
 …
     public double Value {
       get {
+        if (Disabled) return knownValue;
+        else
+          return NumSuccess / (double)(Tries);
+        return NumSuccess / (double)(Tries);
+      }
+    }
     public void UpdateReward(double reward) {
-      Debug.Assert(!Disabled);
       //Debug.Assert(reward.IsAlmost(0.0) || reward.IsAlmost(1.0));
 …
       else NumFailure++;
+    }
-    public void Disable(double reward) {
-      this.NumSuccess = -1;
-      this.NumFailure = -1;
-      this.knownValue = reward;
+    }
     public void Reset() {
       NumSuccess = 0;
       NumFailure = 0;
-      knownValue = 0.0;
+    }
     public void PrintStats() {
       Console.WriteLine("expected value {0,5:F2} disabled {1}", Value, Disabled);
+      Console.WriteLine("expected value {0,5:F2}", Value);
+    }
+  }

r11742	r11832
21	21	foreach (var aInfo in myActionInfos) {
22	22	aIdx++;
23		~~if (aInfo.Disabled) continue;~~
24	23	var theta = Rand.BetaRand(random, aInfo.NumSuccess + alpha, aInfo.NumFailure + beta);
25	24	if (theta > maxTheta) {

r11806	r11832
14	14	public double Value {
15	15	get {
16		return Tries > 0 ? SumReward / Tries : 0.0;
	16	return Tries > 0 ? SumReward / Tries : 0.0;
17	17	}
18	18	}
19	19	public DefaultPolicyActionInfo() {
20		MaxReward = ~~double.MinValue~~;
	20	MaxReward = 0.0;
21	21	}
22	22
…	…
33	33	}
34	34
	35	public override string ToString() {
	36	return string.Format("{0:F3} {1:F3} {2}", Value, MaxReward, Tries);
	37	}
	38
35	39	public static Func<DefaultPolicyActionInfo, double> AverageReward {
36	40	get {

r11742	r11832
31	31	foreach (var aInfo in myActionInfos) {
32	32	aIdx++;
33		~~if (aInfo.Disabled) continue;~~
34	33
35	34	var tries = aInfo.Tries;

-                      r11806
+                      r11832
 namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
   public class MeanAndVariancePolicyActionInfo : IBanditPolicyActionInfo {
-    private bool disabled;
-    public bool Disabled { get { return disabled; } }
     private OnlineMeanAndVarianceEstimator estimator = new OnlineMeanAndVarianceEstimator();
-    private double knownValue;
     public int Tries { get { return estimator.N; } }
     public double SumReward { get { return estimator.Sum; } }
 …
     public double Value {
       get {
+        if (disabled) return knownValue;
+        else
+          return AvgReward;
+        return AvgReward;
+      }
+    }
     public void UpdateReward(double reward) {
-      Debug.Assert(!Disabled);
       estimator.UpdateReward(reward);
+    }
-    public void Disable(double reward) {
-      disabled = true;
-      this.knownValue = reward;
+    }
     public void Reset() {
-      disabled = false;
-      knownValue = 0.0;
       estimator.Reset();
+    }

-                      r11806
+                      r11832
       var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
       int totalTries = myActionInfos.Where(a => !a.Disabled).Sum(a => a.Tries);
+      int totalTries = myActionInfos.Sum(a => a.Tries);
       int aIdx = -1;
 …
       foreach (var aInfo in myActionInfos) {
         aIdx++;
-        if (aInfo.Disabled) continue;
         double q;
         if (aInfo.Tries == 0) {

-                      r11806
+                      r11832
     public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
       var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
       int totalTries = myActionInfos.Where(a => !a.Disabled).Sum(a => a.Tries);
+      int totalTries = myActionInfos.Sum(a => a.Tries);
       double bestQ = double.NegativeInfinity;
       int aIdx = -1;
 …
       foreach (var aInfo in myActionInfos) {
         aIdx++;
-        if (aInfo.Disabled) continue;
         double q;
         if (totalTries <= 1 || aInfo.Tries <= 1 || aInfo.Tries <= Math.Ceiling(8 * Math.Log(totalTries))) {

-                      r11806
+                      r11832
     <Compile Include="Bandits\IBandit.cs" />
     <Compile Include="Bandits\TruncatedNormalBandit.cs" />
+    <Compile Include="GrammarPolicies\GenericFunctionApproximationGrammarPolicy.cs" />
+    <Compile Include="GrammarPolicies\QLearningGrammarPolicy.cs" />
+    <Compile Include="GrammarPolicies\GenericContextualGrammarPolicy.cs" />
     <Compile Include="GrammarPolicies\GenericTDPolicy.cs" />
     <Compile Include="GrammarPolicies\GenericGrammarPolicy.cs">

Note: See TracChangeset for help on using the changeset viewer.