Free cookie consent management tool by TermsFeed Policy Generator

Changeset 12099


Ignore:
Timestamp:
03/02/15 14:08:25 (9 years ago)
Author:
gkronber
Message:

#2283: name for all problems (for output), new unit test, and added testsettings file

Location:
branches/HeuristicLab.Problems.GrammaticalOptimization
Files:
1 added
17 edited

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.Problems.GrammaticalOptimization/GrammaticalOptimization.sln

    r12050 r12099  
    6363Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.MonteCarloTreeSearch", "HeuristicLab.Algorithms.MonteCarloTreeSearch\HeuristicLab.Algorithms.MonteCarloTreeSearch.csproj", "{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}"
    6464EndProject
     65Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D521D944-A271-4A74-8511-6608622509A4}"
     66  ProjectSection(SolutionItems) = preProject
     67    local.testsettings = local.testsettings
     68  EndProjectSection
     69EndProject
    6570Global
    6671  GlobalSection(SolutionConfigurationPlatforms) = preSolution

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/SymbolicRegressionProblem.cs

    r12024 r12099  
    6868    private readonly double[] erc;
    6969    private readonly bool useConstantOpt;
    70 
     70    public string Name { get; private set; }
    7171
    7272    public SymbolicRegressionProblem(Random random, string partOfName, bool useConstantOpt = true) {
    … …  
    8686      var dds = instanceProvider.GetDataDescriptors();
    8787      var problemData = instanceProvider.LoadData(dds.Single(ds => ds.Name.Contains(partOfName)));
     88      this.Name = problemData.Name;
    8889
    8990      this.N = problemData.TrainingIndices.Count();

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IProblem.cs

    r11848 r12099  
    88  // represents a grammatical optimization problem
    99  public interface IProblem {
     10    string Name { get; }
    1011    double BestKnownQuality(int maxLen);
    1112    IGrammar Grammar { get; }

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/EvenParityProblem.cs

    r11857 r12099  
    2727
    2828    private readonly IGrammar grammar;
     29    public string Name { get { return "EvenParity"; } }
     30
    2931    public EvenParityProblem() {
    3032      this.grammar = new Grammar(grammarString);

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/FindPhrasesProblem.cs

    r11865 r12099  
    3131    private readonly SortedSet<string> optimalPhrases;
    3232    private readonly SortedSet<string> decoyPhrases;
     33    public string Name { get { return "FindPhrases"; } }
    3334
    3435    public FindPhrasesProblem(Random rand, int alphabetSize, int numPhrases, int phraseLen, int numOptimalPhrases, int numDecoyPhrases = 1,

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/HardPalindromeProblem.cs

    r11857 r12099  
    1919
    2020    private readonly IGrammar grammar;
     21    public string Name { get { return "HardPalindrome"; } }
     22
    2123    public HardPalindromeProblem() {
    2224      this.grammar = new Grammar(grammarString);

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/PalindromeProblem.cs

    r11857 r12099  
    2121
    2222    private readonly IGrammar grammar;
     23    public string Name { get { return "Palindrome"; } }
    2324    public PalindromeProblem() {
    2425      this.grammar = new Grammar(grammarString);

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/PermutationProblem.cs

    r11972 r12099  
    2020
    2121    private readonly IGrammar grammar;
     22    public string Name { get { return "Permutation"; } }
    2223    public PermutationProblem() {
    2324      this.grammar = new Grammar(grammarString);

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPairProblem.cs

    r11857 r12099  
    2121
    2222    private readonly IGrammar grammar;
     23    public string Name { get { return "RoyalPair"; } }
     24   
    2325    public RoyalPairProblem() {
    2426      this.grammar = new Grammar(grammarString);

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPhraseSequenceProblem.cs

    r11865 r12099  
    3131    private readonly bool phrasesAsSets;
    3232    private readonly SortedSet<string>[] optimalPhrasesForPos;
    33 
     33    public string Name { get { return "RoyalPhraseSequence"; } }
    3434    public RoyalPhraseSequenceProblem(Random rand, int alphabetSize, int sequenceLen, int phraseLen = 1, int numCorrectPhrases = 1, double correctReward = 1.0, double incorrectReward = 0.0, bool phrasesAsSets = false) {
    3535      if (alphabetSize <= 0 || alphabetSize > 26) throw new ArgumentException();

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalRoadProblem.cs

    r11832 r12099  
    1313
    1414    private readonly IGrammar grammar;
     15    public string Name { get { return "RoyalRoad"; } }
    1516    public RoyalRoadProblem() {
    1617      this.grammar = new Grammar(grammarString);

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSequenceProblem.cs

    r11865 r12099  
    2323    private readonly int sequenceLen;
    2424    private readonly SortedSet<char>[] optimalSymbolsForPos;
    25 
     25    public string Name { get { return "RoyalSequence"; } }
    2626    public RoyalSequenceProblem(Random rand, int alphabetSize, int sequenceLen, int k = 1, double correctReward = 1.0, double incorrectReward = 0.0) {
    2727      if (alphabetSize <= 0 || alphabetSize > 26) throw new ArgumentException();

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSymbolProblem.cs

    r11857 r12099  
    2323
    2424    private readonly IGrammar grammar;
     25    public string Name { get { return "RoyalSymbol"; } }
    2526    public RoyalSymbolProblem() {
    2627      this.grammar = new Grammar(grammarString);

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalTreeProblem.cs

    r11972 r12099  
    2727    private readonly IGrammar grammar;
    2828    private readonly char targetRootSymbol;
     29    public string Name { get { return "RoyalTree"; } }
    2930    // the number of levels determines the number of non-terminal symbols
    3031    // non-terminal A has only one argument, non-terminal B has two arguments ...

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/SantaFeAntProblem.cs

    r11981 r12099  
    3232
    3333    private readonly IGrammar grammar;
     34    public string Name { get { return "SantaFeAnt"; } }
    3435
    3536    public SantaFeAntProblem() {

  • branches/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization/Problems/SymbolicRegressionPoly10Problem.cs

    r11976 r12099  
    5454    private readonly double[][] x;
    5555    private readonly double[] y;
     56    public string Name { get { return "SymbolicRegressionPoly10"; } }
    5657
    5758    public SymbolicRegressionPoly10Problem() {

  • branches/HeuristicLab.Problems.GrammaticalOptimization/Test/TestTunedSettings.cs

    r12025 r12099  
    3030    [TestMethod]
    3131    [Timeout(1000 * 60 * 60 * 12)] // 12 hours
    32     // this configuration worked especially well in the experiments
    3332    public void TestAllPoliciesArtificialAnt() {
    3433      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
    … …  
    139138    }
    140139
     140    [TestMethod]
     141    [Timeout(1000 * 60 * 60 * 30)] // 30 hours
     142    public void TestAllPoliciesPoly10() {
     143      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
     144
     145      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
     146      {
     147        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionPoly10Problem(),
     148      };
     149
     150      var policyFactories = new Func<IBanditPolicy>[]
     151        {
     152         () => new RandomPolicy(),
     153         () => new ActiveLearningPolicy(), 
     154         () => new EpsGreedyPolicy(0.01, (aInfo)=> aInfo.MaxReward, "max"),
     155         () => new EpsGreedyPolicy(0.05, (aInfo)=> aInfo.MaxReward, "max"),
     156         () => new EpsGreedyPolicy(0.1, (aInfo)=> aInfo.MaxReward, "max"),
     157         () => new EpsGreedyPolicy(0.2, (aInfo)=> aInfo.MaxReward, "max"),
     158         //() => new GaussianThompsonSamplingPolicy(),
     159         () => new GaussianThompsonSamplingPolicy(true),
     160         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1)),
     161         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1, 1)),
     162         //() => new BernoulliThompsonSamplingPolicy(),
     163         () => new GenericThompsonSamplingPolicy(new BernoulliModel(1, 1)),
     164         () => new EpsGreedyPolicy(0.01),
     165         () => new EpsGreedyPolicy(0.05),
     166         () => new EpsGreedyPolicy(0.1),
     167         () => new EpsGreedyPolicy(0.2),
     168         () => new EpsGreedyPolicy(0.5),
     169         () => new UCTPolicy(0.01),
     170         () => new UCTPolicy(0.05),
     171         () => new UCTPolicy(0.1),
     172         () => new UCTPolicy(0.5),
     173         () => new UCTPolicy(1),
     174         () => new UCTPolicy(2),
     175         () => new UCTPolicy( 5),
     176         () => new UCTPolicy( 10),
     177         () => new ModifiedUCTPolicy(0.01),
     178         () => new ModifiedUCTPolicy(0.05),
     179         () => new ModifiedUCTPolicy(0.1),
     180         () => new ModifiedUCTPolicy(0.5),
     181         () => new ModifiedUCTPolicy(1),
     182         () => new ModifiedUCTPolicy(2),
     183         () => new ModifiedUCTPolicy( 5),
     184         () => new ModifiedUCTPolicy( 10),
     185         () => new UCB1Policy(),
     186         () => new UCB1TunedPolicy(),
     187         () => new UCBNormalPolicy(),
     188         () => new BoltzmannExplorationPolicy(1),
     189         () => new BoltzmannExplorationPolicy(10),
     190         () => new BoltzmannExplorationPolicy(20),
     191         () => new BoltzmannExplorationPolicy(100),
     192         () => new BoltzmannExplorationPolicy(200),
     193         () => new BoltzmannExplorationPolicy(500),
     194         () => new ChernoffIntervalEstimationPolicy( 0.01),
     195         () => new ChernoffIntervalEstimationPolicy( 0.05),
     196         () => new ChernoffIntervalEstimationPolicy( 0.1),
     197         () => new ChernoffIntervalEstimationPolicy( 0.2),
     198         () => new ThresholdAscentPolicy(5, 0.01),
     199         () => new ThresholdAscentPolicy(5, 0.05),
     200         () => new ThresholdAscentPolicy(5, 0.1),
     201         () => new ThresholdAscentPolicy(5, 0.2),
     202         () => new ThresholdAscentPolicy(10, 0.01),
     203         () => new ThresholdAscentPolicy(10, 0.05),
     204         () => new ThresholdAscentPolicy(10, 0.1),
     205         () => new ThresholdAscentPolicy(10, 0.2),
     206         () => new ThresholdAscentPolicy(50, 0.01),
     207         () => new ThresholdAscentPolicy(50, 0.05),
     208         () => new ThresholdAscentPolicy(50, 0.1),
     209         () => new ThresholdAscentPolicy(50, 0.2),
     210         () => new ThresholdAscentPolicy(100, 0.01),
     211         () => new ThresholdAscentPolicy(100, 0.05),
     212         () => new ThresholdAscentPolicy(100, 0.1),
     213         () => new ThresholdAscentPolicy(100, 0.2),
     214         () => new ThresholdAscentPolicy(500, 0.01),
     215         () => new ThresholdAscentPolicy(500, 0.05),
     216         () => new ThresholdAscentPolicy(500, 0.1),
     217         () => new ThresholdAscentPolicy(500, 0.2),
     218         () => new ThresholdAscentPolicy(5000, 0.01),
     219         () => new ThresholdAscentPolicy(10000, 0.01),
     220        };
     221      var maxSizes = new int[] { 23 };  // necessary size symb reg poly 10
     222      int nReps = 20;
     223      int maxIterations = 100000;
     224      foreach (var instanceFactory in instanceFactories) {
     225        var sumBestQ = 0.0;
     226        var sumItersToBest = 0;
     227        double fractionSolved = 0.0;
     228        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
     229          foreach (var policy in policyFactories) {
     230            var prob = conf.Problem;
     231            var maxLen = conf.MaxSize;
     232            var rand = new Random(conf.RandSeed);
     233
     234            var solver = new SequentialSearch(prob, maxLen, rand, 0, new GenericGrammarPolicy(prob, policy(), true));
     235            var problemName = prob.GetType().Name;
     236            var policyName = policy().ToString();
     237            double bestQ; int itersToBest;
     238            RunSolver(solver, problemName, policyName, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
     239            sumBestQ += bestQ;
     240            sumItersToBest += itersToBest;
     241            if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
     242          }
     243        }
     244        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
     245        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
     246        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
     247      }
     248    }
     249
     250    [TestMethod]
     251    [Timeout(1000 * 60 * 60 * 30)] // 30 hours
     252    public void TestAllSymbolicRegression() {
     253      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
     254
     255      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
     256      {
     257        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Nguyen F7", true),
     258        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Keijzer 6", true),
     259        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Vladislavleva-4", true),
     260        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Spatial", true),
     261        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Friedman - II", true),
     262        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Tower", true),
     263      };
     264
     265      var policyFactories = new Func<IBanditPolicy>[]
     266        {
     267         () => new UCTPolicy(0.05),
     268         () => new UCTPolicy(0.1),
     269         () => new ModifiedUCTPolicy(0.01),
     270         () => new ModifiedUCTPolicy(0.05),
     271         () => new UCB1Policy(),
     272         () => new UCB1TunedPolicy(),
     273        };
     274      var maxSizes = new int[] { 20 };  // default limit for all problems
     275      int nReps = 20;
     276      int maxIterations = 10000;
     277      foreach (var instanceFactory in instanceFactories) {
     278        var sumBestQ = 0.0;
     279        var sumItersToBest = 0;
     280        double fractionSolved = 0.0;
     281        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
     282          foreach (var policy in policyFactories) {
     283            var prob = conf.Problem;
     284            var maxLen = conf.MaxSize;
     285            var rand = new Random(conf.RandSeed);
     286
     287            var solver = new SequentialSearch(prob, maxLen, rand, 0, new GenericGrammarPolicy(prob, policy(), true));
     288            var problemName = prob.Name;
     289            var policyName = policy().ToString();
     290            double bestQ; int itersToBest;
     291            RunSolver(solver, problemName, policyName, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
     292            sumBestQ += bestQ;
     293            sumItersToBest += itersToBest;
     294            if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
     295          }
     296        }
     297        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
     298        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
     299        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
     300      }
     301    }
    141302
    142303    [TestMethod]
    … …  
    151312      };
    152313
    153       var maxSizes = new int[] { 23 }; 
     314      var maxSizes = new int[] { 23 };
    154315      int nReps = 20;
    155316      int maxIterations = 100000;
    … …  
    190351      };
    191352
    192       var maxSizes = new int[] { 23 }; 
     353      var maxSizes = new int[] { 23 };
    193354      int nReps = 20;
    194355      int maxIterations = 100000;
Note: See TracChangeset for help on using the changeset viewer.