Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/BernoulliPolicyActionInfo.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/BernoulliPolicyActionInfo.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/BernoulliPolicyActionInfo.cs	(revision 12893)
@@ -12,4 +12,5 @@
     public int NumFailure { get; private set; }
     public int Tries { get { return NumSuccess + NumFailure; } }
+    public double MaxReward { get; private set; }
     public double Value {
       get {
@@ -21,4 +22,5 @@
 
       //if (reward.IsAlmost(1.0)) NumSuccess++;
+      MaxReward = Math.Max(MaxReward, reward);
       if (reward > 0) NumSuccess++;
       else NumFailure++;
@@ -27,4 +29,6 @@
       NumSuccess = 0;
       NumFailure = 0;
+      MaxReward = double.NegativeInfinity;
+
     }
     public void PrintStats() {
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/DefaultPolicyActionInfo.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/DefaultPolicyActionInfo.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/DefaultPolicyActionInfo.cs	(revision 12893)
@@ -22,8 +22,10 @@
     }
 
-    public void UpdateReward(double reward) {
+
+    public void UpdateReward(double reward)
+    {
+      MaxReward = Math.Max(MaxReward, reward);
       Tries++;
       SumReward += reward;
-      MaxReward = Math.Max(MaxReward, reward);
       var delta = reward - avgValue;
       double alpha = 1.0 / Tries;
@@ -34,5 +36,5 @@
       SumReward = 0.0;
       Tries = 0;
-      MaxReward = 0.0;
+      MaxReward = double.NegativeInfinity;
       avgValue = 0.0;
     }
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ExtremeHunterActionInfo.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ExtremeHunterActionInfo.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ExtremeHunterActionInfo.cs	(revision 12893)
@@ -22,5 +22,5 @@
           if (minHeap.Count <= 1) return double.PositiveInfinity;
           double xk = minHeap.GetMin();
-          if (xk.IsAlmost(0.0)) return double.NegativeInfinity;
+          if (xk.IsAlmost(0.0)) return double.PositiveInfinity;
           var alpha = 1.0 / (minHeap.Count - 1) * minHeap.Skip(1).Sum(x => Math.Log(x) - Math.Log(xk));
           Debug.Assert(alpha > 0);
@@ -55,5 +55,5 @@
 
         Debug.Assert(minHeap.Count == ((int)Math.Floor(n * R)));
-        Debug.Assert(maxHeap.Count == 0 || minHeap.Count == 0 || maxHeap.GetMin() < minHeap.GetMin());
+        Debug.Assert(maxHeap.Count == 0 || minHeap.Count == 0 || maxHeap.GetMin() <= minHeap.GetMin());
       }
     }
@@ -64,5 +64,5 @@
     private OnlineHillEstimator hillEstimator;
     private List<double> rewards;
-
+    public double MaxReward { get; private set; }
     public double Value {
       get {
@@ -76,5 +76,7 @@
     public void UpdateReward(double reward) {
       if (reward < 0.0) throw new ArgumentException("reward");
+      MaxReward = Math.Max(MaxReward, reward);
       Tries++;
+      reward = (1 / (1 - reward)); // transformation from [0..1]
       rewards.Add(reward);
       hillEstimator.Update(reward);
@@ -82,4 +84,6 @@
 
     public void Reset() {
+      MaxReward = double.NegativeInfinity;
+
       this.hillEstimator = new OnlineHillEstimator();
       this.rewards = new List<double>();
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/MeanAndVariancePolicyActionInfo.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/MeanAndVariancePolicyActionInfo.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/MeanAndVariancePolicyActionInfo.cs	(revision 12893)
@@ -12,4 +12,5 @@
     public double SumReward { get { return estimator.Sum; } }
     public double AvgReward { get { return estimator.Avg; } }
+    public double MaxReward { get; private set; }
     public double RewardVariance { get { return estimator.Variance; } }
     public double Value {
@@ -20,8 +21,10 @@
 
     public void UpdateReward(double reward) {
+      MaxReward = Math.Max(MaxReward, reward);
       estimator.UpdateReward(reward);
     }
 
     public void Reset() {
+      MaxReward = double.NegativeInfinity;
       estimator.Reset();
     }
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ModelPolicyActionInfo.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ModelPolicyActionInfo.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ModelPolicyActionInfo.cs	(revision 12893)
@@ -10,4 +10,5 @@
   public class ModelPolicyActionInfo : IBanditPolicyActionInfo {
     private readonly IModel model;
+    public double MaxReward { get; private set; }
     public double Value {
       get {
@@ -23,4 +24,5 @@
     public void UpdateReward(double reward) {
       Tries++;
+      MaxReward = Math.Max(MaxReward, reward);
       model.Update(reward);
     }
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/HeuristicLab.Algorithms.Bandits.csproj
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/HeuristicLab.Algorithms.Bandits.csproj	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/HeuristicLab.Algorithms.Bandits.csproj	(revision 12893)
@@ -29,4 +29,5 @@
     <ErrorReport>prompt</ErrorReport>
     <WarningLevel>4</WarningLevel>
+    <UseVSHostingProcess>true</UseVSHostingProcess>
   </PropertyGroup>
   <ItemGroup>
@@ -50,4 +51,6 @@
     <Compile Include="Policies\BoltzmannExplorationPolicy.cs" />
     <Compile Include="Policies\ChernoffIntervalEstimationPolicy.cs" />
+    <Compile Include="Policies\BoltzmannExplorationWithCoolingPolicy.cs" />
+    <Compile Include="Policies\SingleArmPolicy.cs" />
     <Compile Include="Policies\IntervalEstimationPolicy.cs" />
     <Compile Include="Policies\ExtremeHunterPolicy.cs" />
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicyActionInfo.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicyActionInfo.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicyActionInfo.cs	(revision 12893)
@@ -2,4 +2,5 @@
   public interface IBanditPolicyActionInfo {
     //bool Disabled { get; }
+    double MaxReward { get; }
     double Value { get; }
     int Tries { get; }
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ActiveLearningPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ActiveLearningPolicy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ActiveLearningPolicy.cs	(revision 12893)
@@ -31,5 +31,5 @@
           l = double.NegativeInfinity;
         } else {
-          q = aInfo.SumReward / aInfo.Tries;
+          q = aInfo.MaxReward;
           var b = Math.Sqrt(Math.Log(2.0 * k * totalTries / delta) / (2.0 * aInfo.Tries));
           u = q + MaxReward * b;
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationPolicy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationPolicy.cs	(revision 12893)
@@ -12,5 +12,5 @@
     private readonly double beta;
 
-    public BoltzmannExplorationPolicy(double beta)  { 
+    public BoltzmannExplorationPolicy(double beta) {
       if (beta < 0) throw new ArgumentException();
       this.beta = beta;
@@ -19,19 +19,40 @@
       Debug.Assert(actionInfos.Any());
 
-      // select best
       var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
 
       // try any of the untries actions randomly
-      // for RoyalSequence it is much better to select the actions in the order of occurrence (all terminal alternatives first)
-      //if (myActionInfos.Any(aInfo => !aInfo.Disabled && aInfo.Tries == 0)) {
-      //  return myActionInfos
-      //  .Select((aInfo, idx) => new { aInfo, idx })
-      //  .Where(p => !p.aInfo.Disabled)
-      //  .Where(p => p.aInfo.Tries == 0)
-      //  .SelectRandom(random).idx;
+      if (myActionInfos.Any(aInfo => aInfo.Tries == 0)) {
+        return myActionInfos
+        .Select((aInfo, idx) => new { aInfo, idx })
+        .Where(p => p.aInfo.Tries == 0)
+        .SelectRandom(random).idx;
+      }
+
+      // using ranks
+      //var qualities = actionInfos.Select(i => i.MaxReward).ToArray(); // largest reward should have largest rank
+      //var ranks = Enumerable.Range(0, myActionInfos.Count()).ToArray();
+      //Array.Sort(qualities, ranks);
+      //
+      //// set same rank for same quality
+      ////for (int i = 0; i < ranks.Length - 1; i++) {
+      ////  if (qualities[i] == qualities[i + 1]) ranks[i + 1] = ranks[i];
+      ////}
+      ////
+      //
+      //var rankForAction = new int[myActionInfos.Count()];
+      //for (int i = 0; i < rankForAction.Length; i++) {
+      //  rankForAction[ranks[i]] = i;
       //}
+      //
+      //var w = from idx in Enumerable.Range(0, myActionInfos.Count())
+      //        select Math.Exp(beta * rankForAction[idx]);
 
-      var w = from aInfo in myActionInfos
-              select Math.Exp(beta * aInfo.Value);
+
+      // windowing
+      var max = actionInfos.Select(i => i.MaxReward).Max();
+      var min = actionInfos.Select(i => i.MaxReward).Min();
+      double range = max - min;
+      var w = from aInfo in actionInfos
+              select Math.Exp(beta * (aInfo.MaxReward - min) / range);
 
       var bestAction = Enumerable.Range(0, myActionInfos.Count()).SampleProportional(random, w);
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationWithCoolingPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationWithCoolingPolicy.cs	(revision 12893)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationWithCoolingPolicy.cs	(revision 12893)
@@ -0,0 +1,60 @@
+﻿using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  // Reference: Cicirello and Smith, The Max K-armed Bandit: A New Model of Exploration Applied to Search Heuristic Selection, AAAI 2005
+  // uses exponentially decreasing cooling schedule
+  public class BoltzmannExplorationWithCoolingPolicy : IBanditPolicy {
+    private readonly double beta;
+
+    public BoltzmannExplorationWithCoolingPolicy(double beta) {
+      if (beta < 0) throw new ArgumentException();
+      this.beta = beta;
+    }
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      Debug.Assert(actionInfos.Any());
+
+      // select best
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+
+      // try any of the untries actions randomly
+      // for RoyalSequence it is much better to select the actions in the order of occurrence (all terminal alternatives first)
+      if (myActionInfos.Any(aInfo => aInfo.Tries == 0)) {
+        return myActionInfos
+        .Select((aInfo, idx) => new { aInfo, idx })
+        .Where(p => p.aInfo.Tries == 0)
+        .SelectRandom(random).idx;
+      }
+
+      var totalTries = myActionInfos.Sum(i => i.Tries);
+      if (totalTries > 10000) {
+        // take best arm
+        return myActionInfos
+                  .Select((aInfo, idx) => new { aInfo, idx })
+                  .OrderByDescending(t => t.aInfo.MaxReward)
+                  .First().idx;
+      }
+      var w = from aInfo in myActionInfos
+              let q = aInfo.MaxReward // this should be an estimator for the expected maximum of the distribution
+              select Math.Exp(beta * q / Math.Exp(-totalTries / 2000.0));
+
+      var bestAction = Enumerable.Range(0, myActionInfos.Count()).SampleProportional(random, w);
+      Debug.Assert(bestAction >= 0);
+      return bestAction;
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+
+    public override string ToString() {
+      return string.Format("BoltzmannExplorationWithCoolingPolicy({0:F2})", beta);
+    }
+  }
+}
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ChernoffIntervalEstimationPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ChernoffIntervalEstimationPolicy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ChernoffIntervalEstimationPolicy.cs	(revision 12893)
@@ -33,5 +33,5 @@
         } else {
 
-          var avgReward = aInfo.SumReward / aInfo.Tries;
+          var avgReward = aInfo.SumReward / aInfo.Tries;          
 
           // page 5 of "A simple distribution-free approach to the max k-armed bandit problem"
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs	(revision 12893)
@@ -24,7 +24,11 @@
     public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
       Debug.Assert(actionInfos.Any());
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+      int totalTries = myActionInfos.Select(i => i.Tries).Sum();
+      
+      //var eps = Math.Exp(Math.Exp(-totalTries/200.0)) - 1;
+
       if (random.NextDouble() >= eps) { // eps == 0 should be equivalent to pure exploitation, eps == 1 is pure exploration
         // select best
-        var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
         var bestActions = new List<int>();
         double bestQ = double.NegativeInfinity;
@@ -34,5 +38,5 @@
           aIdx++;
 
-          var q = aInfo.Value;
+          var q = aInfo.MaxReward;
 
           if (q > bestQ) {
@@ -45,5 +49,6 @@
         }
         Debug.Assert(bestActions.Any());
-        return bestActions.SelectRandom(random);
+        //return bestActions.SelectRandom(random);
+        return bestActions.First();
       } else {
         // select random
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ExtremeHunterPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ExtremeHunterPolicy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ExtremeHunterPolicy.cs	(revision 12893)
@@ -16,9 +16,11 @@
     public double delta { get; set; }
     public double b { get; set; }
+    public double n { get; set; }
+    public int minPulls { get; set; }
 
-    public ExtremeHunterPolicy(double E = 1.0E-3, double D = 1.0E-2, double b = 1.0) {
+    public ExtremeHunterPolicy(double E = 1.0E-3, double D = 1.0E-2, double b = 1.0, double n = 1.0E4, int minPulls = 100) {
       this.E = E; // parameter TODO
       this.D = D; // parameter TODO
-      this.b = b; // parameter TODO
+      this.b = b; // parameter: set to 1 in original paper "to consider a wide class of distributions"
       // private communication with Alexandra Carpentier: 
       // For instance, on our synthetic experiments, we calibrated the constants by 
@@ -26,4 +28,8 @@
       // out that taking E =  1e-3 is acceptable. For all the datasets 
       // (exact Pareto, approximate Pareto, and network data), we kept this same constant
+
+      // minPulls seems to be set to 100 in the experiments in extreme bandit paper
+      this.minPulls = minPulls; // parameter: TODO (there are conditions for N given in the paper)
+      this.n = n;
     }
 
@@ -31,8 +37,6 @@
       var myActionInfos = actionInfos.OfType<ExtremeHunterActionInfo>();
       double bestQ = double.NegativeInfinity;
-      int totalTries = myActionInfos.Sum(a => a.Tries);
+      // int totalTries = myActionInfos.Sum(a => a.Tries);
       int K = myActionInfos.Count();
-      double n = 1.0E2; // total tries parameter
-      double minPulls = 100; // parameter: TODO (there are conditions for N given in the paper)
 
       this.delta = Math.Exp(-Math.Log(Math.Log(n))) / (2.0 * n * K); // TODO 
@@ -48,8 +52,11 @@
           double t = aInfo.Tries;
           double h = aInfo.Value;
-          var thres = Math.Pow(t, h / (2 * b + 1));
-          double c = Math.Pow(t, 1.0 / (2 * b + 1)) * ((1.0 / t) * aInfo.Rewards.Count(r => r >= thres));
-          q = Math.Pow((c + B2(t)) * n, h + B1(t)) * Gamma(h, B1(t)); // eqn (5)
-          Debug.Assert(q > 0);
+          if (double.IsInfinity(h)) q = 0;
+          else {
+            var thres = Math.Pow(t, h / (2 * b + 1));
+            double c = Math.Pow(t, 1.0 / (2 * b + 1)) * ((1.0 / t) * aInfo.Rewards.Count(r => r >= thres));
+            q = Math.Pow((c + B2(t)) * n, h + B1(t)) * Gamma(h, B1(t)); // eqn (5)
+            Debug.Assert(q > 0);
+          }
         }
         if (q > bestQ) {
@@ -84,5 +91,5 @@
     }
     public override string ToString() {
-      return "ExtremeHunter";
+      return string.Format("ExtremeHunter(E={0:F2},D={1:F2},b={2:F2},n={3:F0},minPulls={4:F0}", E, D, b, n, minPulls);
     }
   }
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/SingleArmPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/SingleArmPolicy.cs	(revision 12893)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/SingleArmPolicy.cs	(revision 12893)
@@ -0,0 +1,29 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class SingleArmPolicy : IBanditPolicy {
+    public int Arm { get; private set; }
+
+    public SingleArmPolicy(int arm) {
+      this.Arm = arm;
+    }
+
+    public override string ToString() {
+      return string.Format("SingleArmPolicy({0})", Arm);
+    }
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      return Arm;
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+  }
+}
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ThresholdAscentPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ThresholdAscentPolicy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ThresholdAscentPolicy.cs	(revision 12893)
@@ -28,5 +28,6 @@
       public int Tries { get; private set; }
       public int thresholdBin = 1;
-
+      //public double MaxReward { get { return Value;  }}
+      public double MaxReward { get; private set; }
       public double Value {
         get {
@@ -37,4 +38,5 @@
 
       public void UpdateReward(double reward) {
+        MaxReward = Math.Max(MaxReward, reward);
         Tries++;
         for (var idx = thresholdBin; idx <= RewardBin(reward); idx++)
@@ -43,4 +45,5 @@
 
       public void Reset() {
+        MaxReward = double.NegativeInfinity;
         Tries = 0;
         thresholdBin = 1;
@@ -84,5 +87,6 @@
       double bestQ = double.NegativeInfinity;
       int k = myActionInfos.Count();
-      var totalTries = myActionInfos.Sum(a => a.Tries);
+      //var totalTries = myActionInfos.Sum(a => a.Tries);
+      var totalTries = 100000;
       int aIdx = -1;
       foreach (var aInfo in myActionInfos) {
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1Policy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1Policy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1Policy.cs	(revision 12893)
@@ -28,5 +28,6 @@
         } else {
 
-          q = aInfo.SumReward / aInfo.Tries + MaxReward * Math.Sqrt((2 * Math.Log(totalTries)) / aInfo.Tries);
+          //q = aInfo.SumReward / aInfo.Tries + MaxReward * Math.Sqrt((2 * Math.Log(totalTries)) / aInfo.Tries);
+          q = aInfo.MaxReward + MaxReward * Math.Sqrt((2 * Math.Log(totalTries)) / aInfo.Tries);
         }
         if (q > bestQ) {
@@ -46,5 +47,5 @@
     }
     public override string ToString() {
-      return "UCB1Policy";
+      return string.Format("UCB1Policy({0})", MaxReward);
     }
   }
Index: branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1TunedPolicy.cs
===================================================================
--- branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1TunedPolicy.cs	(revision 12876)
+++ branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1TunedPolicy.cs	(revision 12893)
@@ -29,4 +29,5 @@
           var tries = aInfo.Tries;
 
+          //var avgReward = aInfo.MaxReward;
           var avgReward = sumReward / tries;
           q = avgReward + Math.Sqrt((Math.Log(totalTries) / tries) * Math.Min(1.0 / 4, V(aInfo, totalTries)));
