source: branches/2895_PushGP_GenealogyAnalysis/HeuristicLab.Tests/Benchmark/GPTests.cs

using System.Linq;
// ReSharper disable PossibleNullReferenceException

namespace HeuristicLab.Tests.Benchmark {
  using System;
  using System.Threading;

  using HeuristicLab.Algorithms.GeneticAlgorithm;
  using HeuristicLab.BenchmarkSuite;
  using HeuristicLab.BenchmarkSuite.Problems;
  using HeuristicLab.Data;
  using HeuristicLab.ParallelEngine;
  using HeuristicLab.Problems.ProgramSynthesis;

  using Microsoft.VisualStudio.TestTools.UnitTesting;

  [TestClass]
  public class GpTests {

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void Checksum() {
      Run<Checksum>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void CollatzNumbers() {
      Run<CollatzNumbers>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void CompareStringLengths() {
      Run<CompareStringLengths>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void CountOdds() {
      Run<CountOdds>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void Digits() {
      Run<Digits>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void DoubleLetters() {
      Run<DoubleLetters>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void EvenSquares() {
      Run<EvenSquares>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void ForLoopIndex() {
      Run<ForLoopIndex>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void Grades() {
      Run<Grades>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void LastIndexOfZero() {
      Run<LastIndexOfZero>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void Median() {
      Run<Median>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void MirrorImage() {
      Run<MirrorImage>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void NegativeToZero() {
      Run<NegativeToZero>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void NumberIo() {
      Run<NumberIO>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void PigLatin() {
      Run<PigLatin>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void ReplaceSpaceWithNewLine() {
      Run<ReplaceSpaceWithNewline>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void ScrabbleScore() {
      Run<ScrabbleScore>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void Smallest() {
      Run<Smallest>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void SmallOrLarge() {
      Run<SmallOrLarge>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void StringDifferences() {
      Run<StringDifferences>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void StringLengthsBackwards() {
      Run<StringLengthsBackwards>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void SumOfSquares() {
      Run<SumOfSquares>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void SuperAnagrams() {
      Run<SuperAnagrams>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void Syllables() {
      Run<Syllables>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void VectorAverage() {
      Run<VectorAverage>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void VectorSummed() {
      Run<VectorSummed>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void WallisPi() {
      Run<WallisPi>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void WordStats() {
      Run<WordStats>();
    }

    [TestMethod]
    [TestProperty("Time", "Long")]
    [TestCategory("ProblemTest")]
    public void XWordLines() {
      Run<XWordLines>();
    }

    static void Run<T>() where T : IBenchmarkSuiteDataDescriptor, new() {
      var trigger = new AutoResetEvent(false);
      var alg = GetAlgorithm(new T());
      Exception exception = null;

      alg.ExceptionOccurred += (sender, args) => {
        exception = args.Value;
        trigger.Set();
      };

      alg.Stopped += (sender, args) => {
        trigger.Set();
      };

      var lastGeneration = -1;
      alg.ExecutionTimeChanged += (sender, args) => {
        if (!alg.Results.Any() || !alg.Results.ContainsKey("CurrentBestQuality")) return;

        var generation = (alg.Results["Generations"].Value as IntValue).Value;

        if (lastGeneration != -1 && generation == lastGeneration) return;

        Console.WriteLine(@"Generation: {0}", generation);

        if (alg.Results.ContainsKey("CurrentBestQuality")) {
          var best = (alg.Results["CurrentBestQuality"].Value as DoubleValue).Value;
          Console.WriteLine(@"CurrentBestQuality: {0}", best);
        }

        if (alg.Results.ContainsKey("CurrentAverageQuality")) {
          var average = (alg.Results["CurrentAverageQuality"].Value as DoubleValue).Value;
          Console.WriteLine(@"CurrentAverageQuality: {0}", average);
        }

        if (alg.Results.ContainsKey("CurrentWorstQuality")) {
          var worst = (alg.Results["CurrentWorstQuality"].Value as DoubleValue).Value;
          Console.WriteLine(@"CurrentWorstQuality: {0}", worst);
        }

        Console.WriteLine();
        lastGeneration = generation;
      };

      alg.Prepare();
      alg.Start();
      trigger.WaitOne();

      if (exception != null) {
        Console.WriteLine(exception.Message);
        throw exception;
      }

      var finalGeneration = (alg.Results["Generations"].Value as IntValue).Value;
      var bestQuality = (alg.Results["CurrentBestQuality"].Value as DoubleValue).Value;
      var averageQuality = (alg.Results["CurrentAverageQuality"].Value as DoubleValue).Value;
      var worstQuality = (alg.Results["CurrentWorstQuality"].Value as DoubleValue).Value;

      Console.WriteLine(@"Final Generation: {0}", finalGeneration);
      Console.WriteLine(@"CurrentBestQuality: {0}", bestQuality);
      Console.WriteLine(@"CurrentAverageQuality: {0}", averageQuality);
      Console.WriteLine(@"CurrentWorstQuality: {0}", worstQuality);
    }

    static GeneticAlgorithm GetAlgorithm(IBenchmarkSuiteDataDescriptor descriptor) {
      var ga = new GeneticAlgorithm();

      var problemData = descriptor.CreateProblemData();
      var pushProblem = new PlushPushBenchmarkSuiteProblem();

      pushProblem.Load(problemData);
      ga.Name = "GA " + pushProblem.Name;

      ga.Problem = pushProblem;

      var crossover = ga.CrossoverParameter.ValidValues.OfType<AlternationCrossover>().First();

      switch (problemData.ProblemType) {
        case ProblemType.NumberIO:
        case ProblemType.SmallOrLarge:
        case ProblemType.Median:
        case ProblemType.Smallest:
          crossover.AlignmentDeviation = 5;
          break;

        default:
          crossover.AlignmentDeviation = 10;
          break;
      }

      crossover.AlternationRate = 0.01;
      ga.Crossover = crossover;

      ga.Selector = ga.SelectorParameter.ValidValues.OfType<LexicaseSelector>().First();

      var mutator = ga.MutatorParameter.ValidValues.OfType<UniformMutation>().First();
      mutator.InstructionMutationProbability = 0.875;
      mutator.CloseMutationProbability = 0.125;
      mutator.CloseIncrementRate = 0.1;
      ga.Mutator = mutator;

      ga.MutationProbability.Value = 0.8;
      ga.SetSeedRandomly.Value = true;
      ga.PopulationSize.Value = 1000;

      ga.MaximumGenerations.Value = 9; // problemData.ProgramExecutionBudget / problemData.TrainingCount / 1000;

      // remove unnecessary analyzers to improve performance
      pushProblem.OperatorsParameter.Value
        .RemoveAll(analyzer =>
          analyzer.GetType() == typeof(IndividualZeroErrorAnalyzer) ||
          analyzer.GetType() == typeof(PushExpressionFrequencyAnalyzer));

      ga.Engine = new ParallelEngine();

      return ga;
    }
  }
}