Changeset 11976 for branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization

Timestamp:

02/11/15 02:22:18 (10 years ago)

Author:

gkronber

Message:

#2283 worked on seq search for ant

Location:

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization

Files:

• SequentialDecisionPolicies/GenericFunctionApproximationGrammarPolicy.cs (modified) (6 diffs)
• Solvers/SequentialSearch.cs (modified) (1 diff)

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GenericFunctionApproximationGrammarPolicy.cs

@@ -14 +14 @@
   public sealed class GenericFunctionApproximationGrammarPolicy : IGrammarPolicy {
     private Dictionary<string, double> featureWeigths; // stores the necessary information for bandit policies for each state (=canonical phrase)
+    private Dictionary<string, int> featureTries;
     private HashSet<string> done;
     private readonly bool useCanonicalPhrases;
     private readonly IProblem problem;
+
 
 
@@ -23 +25 @@
       this.problem = problem;
       this.featureWeigths = new Dictionary<string, double>();
+      this.featureTries = new Dictionary<string, int>();
       this.done = new HashSet<string>();
     }
@@ -57 +60 @@
         originalIdx++;
       }
-      
-      const double beta = 20.0;
-      var w = from q in activeAfterStates
-              select Math.Exp(beta * q);
+
+
+      /*
+      const double beta = 1;
+      var w = from idx in Enumerable.Range(0, maxIdx)
+              let afterStateQ = activeAfterStates[idx]
+              select Math.Exp(beta * afterStateQ);
 
       var bestAction = Enumerable.Range(0, maxIdx).SampleProportional(random, w);
       selectedStateIdx = actionIndexMap[bestAction];
       Debug.Assert(selectedStateIdx >= 0);
-      
-      /*
+      */
+
+
       if (random.NextDouble() < 0.2) {
         selectedStateIdx = actionIndexMap[random.Next(maxIdx)];
@@ -84 +91 @@
         selectedStateIdx = actionIndexMap[bestIdxs[random.Next(bestIdxs.Count)]];
       }
-      */
+
 
 
@@ -114 +121 @@
 
     public int GetTries(string state) {
-      return 1;
+      return 0;
+    }
+
+    public int GetFeatureTries(string featureId) {
+      int t;
+      if (featureTries.TryGetValue(featureId, out t)) {
+        return t;
+      } else return 0;
     }
 
     public double GetValue(string state) {
-      return problem.GetFeatures(state).Sum(feature => GetWeight(feature));
+      return problem.GetFeatures(state).Average(feature => GetWeight(feature));
     }
 
@@ -124 +138 @@
       double w;
       if (featureWeigths.TryGetValue(feature.Id, out w)) return w * feature.Value;
-      else return 0.0; // TODO: alternatives?
+      else return 0.0;
     }
     private void UpdateWeights(string state, double reward) {
-      const double alpha = 0.01;
       double delta = reward - GetValue(state);
+      delta /= problem.GetFeatures(state).Count();
+      const double alpha = 0.001;
       foreach (var feature in problem.GetFeatures(state)) {
+        featureTries[feature.Id] = GetFeatureTries(feature.Id) + 1;
+        Debug.Assert(GetFeatureTries(feature.Id) >= 1);
+        //double alpha = 1.0 / GetFeatureTries(feature.Id);
+        //alpha = Math.Max(alpha, 0.01);
+
         double w;
         if (!featureWeigths.TryGetValue(feature.Id, out w)) {
-          featureWeigths[feature.Id] = alpha * delta;
+          featureWeigths[feature.Id] = alpha * delta * feature.Value;
         } else {
-          featureWeigths[feature.Id] += alpha * delta;
+          featureWeigths[feature.Id] += alpha * delta * feature.Value;
         }
       }

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/SequentialSearch.cs

@@ -166 +166 @@
     private void DistributeReward(double reward) {
       behaviourPolicy.UpdateReward(stateChain, reward);
-      greedyPolicy.UpdateReward(stateChain, reward);
+      //greedyPolicy.UpdateReward(stateChain, reward);
     }