source: branches/HeuristicLab.Problems.GrammaticalOptimization/Test/RunGpExperiments.cs @ 12966

using System;
using System.Collections;
using System.Collections.Generic;
using HeuristicLab.Algorithms.GeneticProgramming;
using HeuristicLab.Algorithms.GrammaticalOptimization;
using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace HeuristicLab.Problems.GrammaticalOptimization.Test {


  [TestClass]
  public class RunGpExperiments {
    private readonly static int[] popSizes = new int[] { 100, 250, 500, 1000, 2500, 5000, 10000 };
    private readonly static double[] mutationRates = new double[] { 0.15 };
    private readonly static int randSeed = 31415;

    internal class GPConfiguration {
      public ISymbolicExpressionTreeProblem Problem;
      public int PopSize;
      public int MaxSize;
      public int RandSeed;
      public double MutationRate;

      public override string ToString() {
        return string.Format("{0} {1} {2} {3} {4}", RandSeed, Problem, PopSize, MaxSize, MutationRate);
      }
    }

    #region permutation
    [TestMethod]
    public void RunStandardGPPermutationProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
      };

      var maxSizes = new int[] { 32 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    [TestMethod]
    public void RunOffspringSelectionGPPermutationProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
      };

      var maxSizes = new int[] { 32 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    #endregion

    #region royalpair
    [TestMethod]
    public void RunStandardGPRoyalPairProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    [TestMethod]
    public void RunOffspringSelectionGPRoyalPairProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    #endregion

    #region royalsymbol
    [TestMethod]
    public void RunStandardGPRoyalSymbolProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    [TestMethod]
    public void RunOffspringSelectionGPRoyalSymbolProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
      };

      var maxSizes = new int[] { 32, 64, 128, 256 };
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    #endregion

    #region findphrases
    [TestMethod]
    public void RunStandardGPFindPhrasesProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
      };

      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    [TestMethod]
    public void RunOffspringSelectionGPFindPhrasesProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
      };

      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
      int nReps = 20;
      int maxIterations = 50000;
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    #endregion

    #region artificial ant
    [TestMethod]
    public void RunStandardGPArtificialAntProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
      int nReps = 20;
      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    [TestMethod]
    public void RunOffspringSelectionGPArtificialAntProblem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
      int nReps = 20;
      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    #endregion

    #region symb-reg-poly-10
    [TestMethod]
    public void RunStandardGPPoly10Problem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
      int nReps = 20;
      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    [TestMethod]
    public void RunOffspringSelectionGPPoly10Problem() {
      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
      {
        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
      };

      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
      int nReps = 20;
      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
      foreach (var instanceFactory in instanceFactories) {
        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
        }
      }
    }
    #endregion

    #region helpers
    private IEnumerable<GPConfiguration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
      int nReps,
      IEnumerable<int> popSizes,
      IEnumerable<int> maxSizes,
      IEnumerable<double> mutationRates) {
      var seedRand = new Random(randSeed);
      // the problem seed is the same for all configuratons 
      // this guarantees that we solve the _same_ problem each time
      // with different solvers and multiple repetitions
      var problemSeed = randSeed;
      for (int i = 0; i < nReps; i++) {
        // in each repetition use the same random seed for all solver configuratons
        // do nReps with different seeds for each configuration
        var solverSeed = seedRand.Next();
        foreach (var popSize in popSizes) {
          foreach (var mutRate in mutationRates) {
            foreach (var maxSize in maxSizes) {
              yield return new GPConfiguration {
                MaxSize = maxSize,
                MutationRate = mutRate,
                PopSize = popSize,
                Problem = problemFactory(problemSeed),
                RandSeed = solverSeed
              };
            }
          }
        }
      }
    }

    private static void RunStandardGpForProblem(
      int randSeed,
      ISymbolicExpressionTreeProblem problem,
      int maxIters,
      int popSize,
      int maxSize,
      double mutationRate
      ) {
      var gp = new StandardGP(problem, new Random(randSeed), false);
      var problemName = problem.GetType().Name;
      var bestKnownQuality = problem.BestKnownQuality(maxSize);
      RunGP(gp, problemName, bestKnownQuality, maxIters, popSize, mutationRate, maxSize);
    }

    private static void RunOffspringSelectionGpForProblem(
  int randSeed,
  ISymbolicExpressionTreeProblem problem,
  int maxIters,
  int popSize,
  int maxSize,
  double mutationRate
  ) {
      var gp = new OffspringSelectionGP(problem, new Random(randSeed), false);
      var problemName = problem.GetType().Name;
      var bestKnownQuality = problem.BestKnownQuality(maxSize);
      RunGP(gp, problemName, bestKnownQuality, maxIters, popSize, mutationRate, maxSize);
    }




    private static void RunGP(IGPSolver gp, string problemName, double bestKnownQuality, int maxIters, int popSize, double mutationRate, int maxSize) {
      int iterations = 0;
      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
      var gpName = gp.GetType().Name;
      gp.SolutionEvaluated += (sentence, quality) => {
        iterations++;
        globalStatistics.AddSentence(sentence, quality);

        if (iterations % 1000 == 0) {
          Console.WriteLine("\"{0,25}\" {1} {2:N2} {3} \"{4,25}\" {5}", gpName, popSize, mutationRate, maxSize, problemName, globalStatistics);
        }
      };

      gp.PopulationSize = popSize;
      gp.MutationRate = mutationRate;
      gp.MaxSolutionSize = maxSize + 2;
      gp.MaxSolutionDepth = maxSize + 2;

      gp.Run(maxIters);
    }
    #endregion
  }
}