source: branches/HeuristicLab.Problems.GrammaticalOptimization/Test/RunMctsExperiments.cs @ 13777

using System;
using System.Collections;
using System.Collections.Generic;
using HeuristicLab.Algorithms.Bandits.BanditPolicies;
using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
using HeuristicLab.Algorithms.GeneticProgramming;
using HeuristicLab.Algorithms.GrammaticalOptimization;
using HeuristicLab.Problems.GrammaticalOptimization;
using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace HeuristicLab.Problems.GrammaticalOptimization.Test {

  [TestClass]
  public class RunMctsExperiments {
    private readonly static int randSeed = 31415;

    internal class Configuration {
      public ISymbolicExpressionTreeProblem Problem;
      public int MaxSize;
      public int RandSeed;

      public override string ToString() {
        return string.Format("{0} {1} {2}", RandSeed, Problem, MaxSize);
      }
    }


    #region artificial ant
    [TestMethod]
    public void RunSeqSolvFuncApproxArtificialAntProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
      };

      var maxSizes = new int[] { 17 }; // size of sequential representation is 17
      int nReps = 30;
      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunMctsForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }

    #endregion

    #region symb-reg-poly-10
    [TestMethod]
    public void RunSeqSolvFuncApproxPoly10Problem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
      };

      var maxSizes = new int[] { 23 }; // size of sequential representation is 23
      int nReps = 30;
      int maxIterations = 200000; // sequentialsearch should find the optimum within 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunMctsForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }

    #endregion

    #region helpers
    private IEnumerable<Configuration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
      int nReps,
      IEnumerable<int> maxSizes
      ) {
      var seedRand = new Random(randSeed);
      // the problem seed is the same for all configuratons 
      // this guarantees that we solve the _same_ problem each time
      // with different solvers and multiple repetitions
      var problemSeed = randSeed;
      for (int i = 0; i < nReps; i++) {
        // in each repetition use the same random seed for all solver configuratons
        // do nReps with different seeds for each configuration
        var solverSeed = seedRand.Next();
        foreach (var maxSize in maxSizes) {
          yield return new Configuration {
            MaxSize = maxSize,
            Problem = problemFactory(problemSeed),
            RandSeed = solverSeed
          };
        }
      }
    }

    private static void RunMctsForProblem(
      int randSeed,
      IProblem problem,
      int maxIters,
      int maxSize
      ) {
      var solver = new SequentialSearch(problem, maxSize, new Random(randSeed), 0,
        new GenericGrammarPolicy(problem, new UCB1Policy(), false));
      var problemName = problem.GetType().Name;
      var bestKnownQuality = problem.BestKnownQuality(maxSize);
      RunSolver(solver, problemName, bestKnownQuality, maxIters, maxSize);
    }

    private static void RunSolver(ISolver solver, string problemName, double bestKnownQuality, int maxIters, int maxSize) {
      int iterations = 0;
      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
      var gpName = solver.GetType().Name;
      solver.SolutionEvaluated += (sentence, quality) => {
        iterations++;
        globalStatistics.AddSentence(sentence, quality);

        if (iterations % 1000 == 0) {
          Console.WriteLine("\"{0,25}\" {1} \"{2,25}\" {3}", gpName, maxSize, problemName, globalStatistics);
        }
      };

      solver.Run(maxIters);
    }
    #endregion
  }
}