Changeset 11742 for branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization/MctsSampler.cs

Timestamp:

01/09/15 14:57:28 (10 years ago)

Author:

gkronber

Message:

#2283 refactoring

File:

• branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization/MctsSampler.cs (modified) (11 diffs)

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization/MctsSampler.cs

@@ -12 +12 @@
       public string ident;
       public int randomTries;
-      public int policyTries;
-      public IPolicyActionInfo actionInfo;
+      public IBanditPolicyActionInfo actionInfo;
       public TreeNode[] children;
       public bool done = false;
@@ -22 +21 @@
 
       public override string ToString() {
-        return string.Format("Node({0} tries: {1}, done: {2}, policy: {3})", ident, randomTries + policyTries, done, actionInfo);
+        return string.Format("Node({0} tries: {1}, done: {2}, policy: {3})", ident, actionInfo.Tries, done, actionInfo);
       }
     }
@@ -34 +33 @@
     private readonly Random random;
     private readonly int randomTries;
-    private readonly IPolicy policy;
+    private readonly IBanditPolicy policy;
 
     private List<TreeNode> updateChain;
@@ -47 +46 @@
     // }
 
-    public MctsSampler(IProblem problem, int maxLen, Random random, int randomTries, IPolicy policy) {
+    public MctsSampler(IProblem problem, int maxLen, Random random, int randomTries, IBanditPolicy policy) {
       this.maxLen = maxLen;
       this.problem = problem;
@@ -78 +77 @@
     public void PrintStats() {
       var n = rootNode;
-      Console.WriteLine("depth: {0,5} size: {1,10} root tries {2,10}", treeDepth, treeSize, rootNode.policyTries + rootNode.randomTries);
+      Console.WriteLine("depth: {0,5} size: {1,10} root tries {2,10}", treeDepth, treeSize, n.actionInfo.Tries);
       while (n.children != null) {
         Console.WriteLine();
         Console.WriteLine("{0,5}->{1,-50}", n.ident, string.Join(" ", n.children.Select(ch => string.Format("{0,4}", ch.ident))));
-        Console.WriteLine("{0,5}  {1,-50}", string.Empty, string.Join(" ", n.children.Select(ch => string.Format("{0,4}", ch.randomTries + ch.policyTries))));
+        Console.WriteLine("{0,5}  {1,-50}", string.Empty, string.Join(" ", n.children.Select(ch => string.Format("{0,4:F2}", ch.actionInfo.Value * 10))));
+        Console.WriteLine("{0,5}  {1,-50}", string.Empty, string.Join(" ", n.children.Select(ch => string.Format("{0,4}", ch.done ? "X" : ch.actionInfo.Tries.ToString()))));
         //n.policy.PrintStats();
-        n = n.children.OrderByDescending(c => c.policyTries).First();
+        n = n.children.Where(ch => !ch.done).OrderByDescending(c => c.actionInfo.Value).First();
       }
       Console.ReadLine();
@@ -108 +108 @@
       if (g.MinPhraseLength(phrase) > maxLen) throw new ArgumentException();
       TreeNode n = rootNode;
-      bool done = phrase.IsTerminal;
       var curDepth = 0;
-      while (!done) {
+      while (!phrase.IsTerminal) {
         updateChain.Add(n);
 
@@ -127 +126 @@
           if (n.randomTries == randomTries && n.children == null) {
             n.children = alts.Select(alt => new TreeNode(alt.ToString())).ToArray(); // create a new node for each alternative
-            //n.children = alts.Select(alt => new TreeNode(string.Empty)).ToArray(); // create a new node for each alternative
             foreach (var ch in n.children) ch.actionInfo = policy.CreateActionInfo();
             treeSize += n.children.Length;
           }
-          n.policyTries++;
           // => select using bandit policy
           int selectedAltIdx = policy.SelectAction(random, n.children.Select(c => c.actionInfo));
@@ -140 +137 @@
 
           curDepth++;
-
-          done = phrase.IsTerminal;
 
           // prepare for next iteration
@@ -153 +148 @@
       // the last node is a leaf node (sentence is done), so we never need to visit this node again
       n.done = true;
-      n.actionInfo.Disable();
 
       treeDepth = Math.Max(treeDepth, curDepth);
@@ -165 +159 @@
       foreach (var e in updateChain) {
         var node = e;
+        if (node.done) node.actionInfo.Disable();
         if (node.children != null && node.children.All(c => c.done)) {
           node.done = true;
@@ -171 +166 @@
         if (!node.done) {
           node.actionInfo.UpdateReward(reward);
-          //policy.UpdateReward(action, reward / updateChain.Count);
         }
       }