source: branches/PushGP/HeuristicLab.Algorithms.PushGP/HeuristicLab.Tests/Benchmark/ProblemTests.cs @ 14602

namespace HeuristicLab.Tests.Benchmark.Problem {
  using System;
  using System.Linq;
  using System.Threading.Tasks;

  using HeuristicLab.Algorithms.PushGP.Data.Random;
  using HeuristicLab.Algorithms.PushGP.Expressions;
  using HeuristicLab.Algorithms.PushGP.Generators;
  using HeuristicLab.Algorithms.PushGP.Interpreter;
  using HeuristicLab.BenchmarkSuite;
  using HeuristicLab.BenchmarkSuite.Problems;
  using HeuristicLab.Problems.Instances;

  using Microsoft.VisualStudio.TestTools.UnitTesting;

  [TestClass]
  public class ProblemTests {

    private void Test(params IDataDescriptor[] descriptors) {
      var instance = new BenchmarkSuitePushInstanceProvider();

      foreach (var d in descriptors) {
        var examples = instance.LoadData(d);
      }
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void CountOdds() {
      RandomWalk(new CountOdds());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void Checksum() {
      RandomWalk(new Checksum());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void CollatzNumbers() {
      RandomWalk(new CollatzNumbers());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void CompareStringLengths() {
      RandomWalk(new CompareStringLengths());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void Digits() {
      RandomWalk(new Digits());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void DoubleLetters() {
      RandomWalk(new DoubleLetters());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void EvenSquares() {
      RandomWalk(new EvenSquares());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void ForLoopIndex() {
      RandomWalk(new ForLoopIndex());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void Grades() {
      RandomWalk(new Grades());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void LastIndexOfZero() {
      RandomWalk(new LastIndexOfZero());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void Median() {
      RandomWalk(new Median());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void MirrorImage() {
      RandomWalk(new MirrorImage());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void NegativeToZero() {
      RandomWalk(new NegativeToZero());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void NumberIo() {
      RandomWalk(new NumberIo());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void PigLatin() {
      RandomWalk(new PigLatin());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void ReplaceSpaceWithNewLine() {
      RandomWalk(new ReplaceSpaceWithNewline());
    }

    private static void RandomWalk(IDataDescriptor descriptor) {
      var maxProgramSizeLimit = 1024;
      var iterations = 1; //4000;
      var best = double.MaxValue;
      var globalExecCounter = 0;
      var lockObj = new object();
      var lockCount = new object();
      RandomFactory.Seed = 1337;

      Expression bestProgram = null;
      var config = new Configuration { EvalPushLimit = 500 };
      var pool = new PushGpInterpreterPool(config);

      var instance = new BenchmarkSuitePushInstanceProvider();
      var examples = instance.LoadData(descriptor);

      Parallel.For(0, iterations, i => {
        var execCounter = 0;
        var program = CodeGenerator.RandomProgram(maxProgramSizeLimit);
        var results = new double[examples.TrainingCount];

        using (var interpreter = pool.GetInstance()) {
          for (var j = 0; j < examples.TrainingCount; j++) {
            examples.ApplyTrainingInput(interpreter, j);
            interpreter.Run(program);

            results[j] = examples.CompareTrainingOutput(interpreter, j);

            execCounter += interpreter.ExecCounter;
            interpreter.Clear();
          }
        }

        lock (lockCount) {
          globalExecCounter += execCounter;
        }

        var avg = results.Average();

        if (avg >= best) return;

        lock (lockObj) {
          if (avg < best) {
            best = avg;
            bestProgram = program;
          }
        }
      });

      var resultsTest = new double[examples.TrainingCount];
      Parallel.For(0, examples.TrainingCount, i => {
        using (var interpreter = pool.GetInstance()) {
          examples.ApplyTestInput(interpreter, i);
          interpreter.Run(bestProgram);

          resultsTest[i] = examples.CompareTestOutput(interpreter, i);
        }
      });

      var averageTestResult = resultsTest.Average();

      Console.WriteLine("Best Training: {0}", best);
      Console.WriteLine("Test: {0}", averageTestResult);
      Console.WriteLine("Best trainig program: {0}", bestProgram);
      Console.WriteLine("ExecCounter: {0}", globalExecCounter);
    }
  }
}