source: branches/HeuristicLab.Problems.GrammaticalOptimization/Test/RunBaselineExperiments.cs @ 12014

using System;
using System.Collections;
using System.Collections.Generic;
using HeuristicLab.Algorithms.Bandits.BanditPolicies;
using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
using HeuristicLab.Algorithms.GeneticProgramming;
using HeuristicLab.Algorithms.GrammaticalOptimization;
using HeuristicLab.Problems.GrammaticalOptimization;
using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using RandomPolicy = HeuristicLab.Algorithms.Bandits.BanditPolicies.RandomPolicy;

namespace HeuristicLab.Problems.GrammaticalOptimization.Test {


  [TestClass]
  public class RunBaselineExperiments {
    private readonly static int randSeed = 31415;

    internal class Configuration {
      public ISymbolicExpressionTreeProblem Problem;
      public int MaxSize;
      public int RandSeed;

      public override string ToString() {
        return string.Format("{0} {1} {2}", RandSeed, Problem, MaxSize);
      }
    }

    #region permutation
    [TestMethod]
    public void RunPureRandomSearchPermutationProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
      };

      var maxSizes = new int[] { 32 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    [TestMethod]
    public void RunRandomSearchNoResamplingPermutationProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
      };

      var maxSizes = new int[] { 32 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    #endregion

    #region royalpair
    [TestMethod]
    public void RunPureRandomSearchRoyalPairProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    [TestMethod]
    public void RunRandomSearchNoResamplingRoyalPairProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    #endregion

    #region royalsymbol
    [TestMethod]
    public void RunPureRandomSearchRoyalSymbolProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    [TestMethod]
    public void RunRandomSearchNoResamplingRoyalSymbolProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    #endregion

    #region findphrases
    [TestMethod]
    public void RunPureRandomSearchFindPhrasesProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
      };

      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    [TestMethod]
    public void RunRandomSearchNoResamplingFindPhrasesProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
      };

      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }

    #endregion

    #region artificial ant
    [TestMethod]
    public void RunPureRandomSearchArtificialAntProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
      int nReps = 20;
      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    [TestMethod]
    public void RunRandomSearchNoResamplingArtificialAntProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
      int nReps = 20;
      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    #endregion

    #region symb-reg-poly-10
    [TestMethod]
    public void RunPureRandomSearchPoly10Problem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
      int nReps = 20;
      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    [TestMethod]
    public void RunRandomSearchNoResamplingPoly10Problem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
      int nReps = 20;
      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
        }
      }
    }
    #endregion

    #region helpers
    private IEnumerable<Configuration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
      int nReps,
      IEnumerable<int> maxSizes
      ) {
      var seedRand = new Random(randSeed);
      // the problem seed is the same for all configuratons 
      // this guarantees that we solve the _same_ problem each time
      // with different solvers and multiple repetitions
      var problemSeed = randSeed;
      for (int i = 0; i < nReps; i++) {
        // in each repetition use the same random seed for all solver configuratons
        // do nReps with different seeds for each configuration
        var solverSeed = seedRand.Next();
        foreach (var maxSize in maxSizes) {
          yield return new Configuration {
            MaxSize = maxSize,
            Problem = problemFactory(problemSeed),
            RandSeed = solverSeed
          };
        }
      }
    }

    private static void RunPureRandomSearchForProblem(
      int randSeed,
      ISymbolicExpressionTreeProblem problem,
      int maxIters,
      int maxSize
      ) {
      var solver = new RandomSearch(problem, new Random(randSeed), maxSize);
      var problemName = problem.GetType().Name;
      var bestKnownQuality = problem.BestKnownQuality(maxSize);
      RunSolver(solver, problemName, bestKnownQuality, maxIters, maxSize);
    }

    private static void RunRandomSearchNoResamplingForProblem(
  int randSeed,
  ISymbolicExpressionTreeProblem problem,
  int maxIters,
  int maxSize
  ) {
      var solver = new SequentialSearch(problem, maxSize, new Random(randSeed), 0,
        new GenericGrammarPolicy(problem, new RandomPolicy(), false));

      var problemName = problem.GetType().Name;
      var bestKnownQuality = problem.BestKnownQuality(maxSize);
      RunSolver(solver, problemName, bestKnownQuality, maxIters, maxSize);
    }

    private static void RunSolver(ISolver solver, string problemName, double bestKnownQuality, int maxIters, int maxSize) {
      int iterations = 0;
      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
      var gpName = solver.GetType().Name;
      solver.SolutionEvaluated += (sentence, quality) => {
        iterations++;
        globalStatistics.AddSentence(sentence, quality);

        if (iterations % 1000 == 0) {
          Console.WriteLine("\"{0,25}\" {1} \"{2,25}\" {3}", gpName, maxSize, problemName, globalStatistics);
        }
      };

      solver.Run(maxIters);
    }
    #endregion
  }
}