Changeset 12099 for branches/HeuristicLab.Problems.GrammaticalOptimization

Timestamp:

03/02/15 14:08:25 (10 years ago)

Author:

gkronber

Message:

#2283: name for all problems (for output), new unit test, and added testsettings file

Location:

branches/HeuristicLab.Problems.GrammaticalOptimization

Files:

• GrammaticalOptimization.sln (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization.SymbReg/SymbolicRegressionProblem.cs (modified) (2 diffs)
• HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/EvenParityProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/FindPhrasesProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/HardPalindromeProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/PalindromeProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/PermutationProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPairProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPhraseSequenceProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalRoadProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSequenceProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSymbolProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalTreeProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/SantaFeAntProblem.cs (modified) (1 diff)
• HeuristicLab.Problems.GrammaticalOptimization/Problems/SymbolicRegressionPoly10Problem.cs (modified) (1 diff)
• Test/TestTunedSettings.cs (modified) (4 diffs)
• local.testsettings (added)

branches/HeuristicLab.Problems.GrammaticalOptimization/GrammaticalOptimization.sln

@@ -63 +63 @@
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.MonteCarloTreeSearch", "HeuristicLab.Algorithms.MonteCarloTreeSearch\HeuristicLab.Algorithms.MonteCarloTreeSearch.csproj", "{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D521D944-A271-4A74-8511-6608622509A4}"
+  ProjectSection(SolutionItems) = preProject
+    local.testsettings = local.testsettings
+  EndProjectSection
+EndProject
 Global
   GlobalSection(SolutionConfigurationPlatforms) = preSolution

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/SymbolicRegressionProblem.cs

@@ -68 +68 @@
     private readonly double[] erc;
     private readonly bool useConstantOpt;
-
+    public string Name { get; private set; }
 
     public SymbolicRegressionProblem(Random random, string partOfName, bool useConstantOpt = true) {
@@ -86 +86 @@
       var dds = instanceProvider.GetDataDescriptors();
       var problemData = instanceProvider.LoadData(dds.Single(ds => ds.Name.Contains(partOfName)));
+      this.Name = problemData.Name;
 
       this.N = problemData.TrainingIndices.Count();

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IProblem.cs

@@ -8 +8 @@
   // represents a grammatical optimization problem
   public interface IProblem {
+    string Name { get; }
     double BestKnownQuality(int maxLen);
     IGrammar Grammar { get; }

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/EvenParityProblem.cs

@@ -27 +27 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "EvenParity"; } }
+
     public EvenParityProblem() {
       this.grammar = new Grammar(grammarString);

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/FindPhrasesProblem.cs

@@ -31 +31 @@
     private readonly SortedSet<string> optimalPhrases;
     private readonly SortedSet<string> decoyPhrases;
+    public string Name { get { return "FindPhrases"; } }
 
     public FindPhrasesProblem(Random rand, int alphabetSize, int numPhrases, int phraseLen, int numOptimalPhrases, int numDecoyPhrases = 1,

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/HardPalindromeProblem.cs

@@ -19 +19 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "HardPalindrome"; } }
+
     public HardPalindromeProblem() {
       this.grammar = new Grammar(grammarString);

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/PalindromeProblem.cs

@@ -21 +21 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "Palindrome"; } }
     public PalindromeProblem() {
       this.grammar = new Grammar(grammarString);

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/PermutationProblem.cs

@@ -20 +20 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "Permutation"; } }
     public PermutationProblem() {
       this.grammar = new Grammar(grammarString);

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPairProblem.cs

@@ -21 +21 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "RoyalPair"; } }
+    
     public RoyalPairProblem() {
       this.grammar = new Grammar(grammarString);

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPhraseSequenceProblem.cs

@@ -31 +31 @@
     private readonly bool phrasesAsSets;
     private readonly SortedSet<string>[] optimalPhrasesForPos;
-
+    public string Name { get { return "RoyalPhraseSequence"; } }
     public RoyalPhraseSequenceProblem(Random rand, int alphabetSize, int sequenceLen, int phraseLen = 1, int numCorrectPhrases = 1, double correctReward = 1.0, double incorrectReward = 0.0, bool phrasesAsSets = false) {
       if (alphabetSize <= 0 || alphabetSize > 26) throw new ArgumentException();

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalRoadProblem.cs

@@ -13 +13 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "RoyalRoad"; } }
     public RoyalRoadProblem() {
       this.grammar = new Grammar(grammarString);

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSequenceProblem.cs

@@ -23 +23 @@
     private readonly int sequenceLen;
     private readonly SortedSet<char>[] optimalSymbolsForPos;
-
+    public string Name { get { return "RoyalSequence"; } }
     public RoyalSequenceProblem(Random rand, int alphabetSize, int sequenceLen, int k = 1, double correctReward = 1.0, double incorrectReward = 0.0) {
       if (alphabetSize <= 0 || alphabetSize > 26) throw new ArgumentException();

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSymbolProblem.cs

@@ -23 +23 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "RoyalSymbol"; } }
     public RoyalSymbolProblem() {
       this.grammar = new Grammar(grammarString);

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalTreeProblem.cs

@@ -27 +27 @@
     private readonly IGrammar grammar;
     private readonly char targetRootSymbol;
+    public string Name { get { return "RoyalTree"; } }
     // the number of levels determines the number of non-terminal symbols
     // non-terminal A has only one argument, non-terminal B has two arguments ...

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/SantaFeAntProblem.cs

@@ -32 +32 @@
 
     private readonly IGrammar grammar;
+    public string Name { get { return "SantaFeAnt"; } }
 
     public SantaFeAntProblem() {

branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/SymbolicRegressionPoly10Problem.cs

@@ -54 +54 @@
     private readonly double[][] x;
     private readonly double[] y;
+    public string Name { get { return "SymbolicRegressionPoly10"; } }
 
     public SymbolicRegressionPoly10Problem() {

branches/HeuristicLab.Problems.GrammaticalOptimization/Test/TestTunedSettings.cs

@@ -30 +30 @@
     [TestMethod]
     [Timeout(1000 * 60 * 60 * 12)] // 12 hours
-    // this configuration worked especially well in the experiments
     public void TestAllPoliciesArtificialAnt() {
       CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
@@ -139 +138 @@
     }
 
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 30)] // 30 hours
+    public void TestAllPoliciesPoly10() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var policyFactories = new Func<IBanditPolicy>[]
+        {
+         () => new RandomPolicy(), 
+         () => new ActiveLearningPolicy(),  
+         () => new EpsGreedyPolicy(0.01, (aInfo)=> aInfo.MaxReward, "max"), 
+         () => new EpsGreedyPolicy(0.05, (aInfo)=> aInfo.MaxReward, "max"), 
+         () => new EpsGreedyPolicy(0.1, (aInfo)=> aInfo.MaxReward, "max"), 
+         () => new EpsGreedyPolicy(0.2, (aInfo)=> aInfo.MaxReward, "max"), 
+         //() => new GaussianThompsonSamplingPolicy(), 
+         () => new GaussianThompsonSamplingPolicy(true), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1)), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1, 1)), 
+         //() => new BernoulliThompsonSamplingPolicy(),
+         () => new GenericThompsonSamplingPolicy(new BernoulliModel(1, 1)), 
+         () => new EpsGreedyPolicy(0.01), 
+         () => new EpsGreedyPolicy(0.05), 
+         () => new EpsGreedyPolicy(0.1), 
+         () => new EpsGreedyPolicy(0.2), 
+         () => new EpsGreedyPolicy(0.5), 
+         () => new UCTPolicy(0.01),
+         () => new UCTPolicy(0.05),
+         () => new UCTPolicy(0.1),
+         () => new UCTPolicy(0.5),
+         () => new UCTPolicy(1),
+         () => new UCTPolicy(2),
+         () => new UCTPolicy( 5),
+         () => new UCTPolicy( 10),
+         () => new ModifiedUCTPolicy(0.01),
+         () => new ModifiedUCTPolicy(0.05),
+         () => new ModifiedUCTPolicy(0.1),
+         () => new ModifiedUCTPolicy(0.5),
+         () => new ModifiedUCTPolicy(1),
+         () => new ModifiedUCTPolicy(2),
+         () => new ModifiedUCTPolicy( 5),
+         () => new ModifiedUCTPolicy( 10),
+         () => new UCB1Policy(), 
+         () => new UCB1TunedPolicy(), 
+         () => new UCBNormalPolicy(), 
+         () => new BoltzmannExplorationPolicy(1),
+         () => new BoltzmannExplorationPolicy(10),
+         () => new BoltzmannExplorationPolicy(20),
+         () => new BoltzmannExplorationPolicy(100),
+         () => new BoltzmannExplorationPolicy(200),
+         () => new BoltzmannExplorationPolicy(500),
+         () => new ChernoffIntervalEstimationPolicy( 0.01), 
+         () => new ChernoffIntervalEstimationPolicy( 0.05),
+         () => new ChernoffIntervalEstimationPolicy( 0.1),
+         () => new ChernoffIntervalEstimationPolicy( 0.2),
+         () => new ThresholdAscentPolicy(5, 0.01), 
+         () => new ThresholdAscentPolicy(5, 0.05),
+         () => new ThresholdAscentPolicy(5, 0.1),
+         () => new ThresholdAscentPolicy(5, 0.2),
+         () => new ThresholdAscentPolicy(10, 0.01), 
+         () => new ThresholdAscentPolicy(10, 0.05),
+         () => new ThresholdAscentPolicy(10, 0.1),
+         () => new ThresholdAscentPolicy(10, 0.2),
+         () => new ThresholdAscentPolicy(50, 0.01), 
+         () => new ThresholdAscentPolicy(50, 0.05),
+         () => new ThresholdAscentPolicy(50, 0.1),
+         () => new ThresholdAscentPolicy(50, 0.2),
+         () => new ThresholdAscentPolicy(100, 0.01), 
+         () => new ThresholdAscentPolicy(100, 0.05),
+         () => new ThresholdAscentPolicy(100, 0.1),
+         () => new ThresholdAscentPolicy(100, 0.2),
+         () => new ThresholdAscentPolicy(500, 0.01), 
+         () => new ThresholdAscentPolicy(500, 0.05),
+         () => new ThresholdAscentPolicy(500, 0.1),
+         () => new ThresholdAscentPolicy(500, 0.2),
+         () => new ThresholdAscentPolicy(5000, 0.01), 
+         () => new ThresholdAscentPolicy(10000, 0.01), 
+        };
+      var maxSizes = new int[] { 23 };  // necessary size symb reg poly 10
+      int nReps = 20;
+      int maxIterations = 100000;
+      foreach (var instanceFactory in instanceFactories) {
+        var sumBestQ = 0.0;
+        var sumItersToBest = 0;
+        double fractionSolved = 0.0;
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          foreach (var policy in policyFactories) {
+            var prob = conf.Problem;
+            var maxLen = conf.MaxSize;
+            var rand = new Random(conf.RandSeed);
+
+            var solver = new SequentialSearch(prob, maxLen, rand, 0, new GenericGrammarPolicy(prob, policy(), true));
+            var problemName = prob.GetType().Name;
+            var policyName = policy().ToString();
+            double bestQ; int itersToBest;
+            RunSolver(solver, problemName, policyName, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
+            sumBestQ += bestQ;
+            sumItersToBest += itersToBest;
+            if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
+          }
+        }
+        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
+        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
+        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
+      }
+    }
+
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 30)] // 30 hours
+    public void TestAllSymbolicRegression() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Nguyen F7", true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Keijzer 6", true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Vladislavleva-4", true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Spatial", true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Friedman - II", true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Tower", true), 
+      };
+
+      var policyFactories = new Func<IBanditPolicy>[]
+        {
+         () => new UCTPolicy(0.05),
+         () => new UCTPolicy(0.1),
+         () => new ModifiedUCTPolicy(0.01),
+         () => new ModifiedUCTPolicy(0.05),
+         () => new UCB1Policy(), 
+         () => new UCB1TunedPolicy(), 
+        };
+      var maxSizes = new int[] { 20 };  // default limit for all problems
+      int nReps = 20;
+      int maxIterations = 10000;
+      foreach (var instanceFactory in instanceFactories) {
+        var sumBestQ = 0.0;
+        var sumItersToBest = 0;
+        double fractionSolved = 0.0;
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          foreach (var policy in policyFactories) {
+            var prob = conf.Problem;
+            var maxLen = conf.MaxSize;
+            var rand = new Random(conf.RandSeed);
+
+            var solver = new SequentialSearch(prob, maxLen, rand, 0, new GenericGrammarPolicy(prob, policy(), true));
+            var problemName = prob.Name;
+            var policyName = policy().ToString();
+            double bestQ; int itersToBest;
+            RunSolver(solver, problemName, policyName, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
+            sumBestQ += bestQ;
+            sumItersToBest += itersToBest;
+            if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
+          }
+        }
+        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
+        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
+        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
+      }
+    }
 
     [TestMethod]
@@ -151 +312 @@
       };
 
-      var maxSizes = new int[] { 23 };  
+      var maxSizes = new int[] { 23 };
       int nReps = 20;
       int maxIterations = 100000;
@@ -190 +351 @@
       };
 
-      var maxSizes = new int[] { 23 };  
+      var maxSizes = new int[] { 23 };
       int nReps = 20;
       int maxIterations = 100000;