source: branches/PushGP/HeuristicLab.PushGP/HeuristicLab.Tests/Benchmark/ProblemTests.cs @ 14733

namespace HeuristicLab.Tests.Benchmark.Problem {
  using System;
  using System.Collections.Generic;
  using System.Linq;
  using System.Threading.Tasks;

  using HeuristicLab.BenchmarkSuite;
  using HeuristicLab.BenchmarkSuite.Problems;
  using HeuristicLab.Problems.Instances;
  using HeuristicLab.Problems.ProgramSynthesis.Push.Configuration;
  using HeuristicLab.Problems.ProgramSynthesis.Push.Expressions;
  using HeuristicLab.Problems.ProgramSynthesis.Push.Generators;
  using HeuristicLab.Problems.ProgramSynthesis.Push.Interpreter;
  using HeuristicLab.Problems.ProgramSynthesis.Push.Stack;
  using HeuristicLab.Random;

  using Microsoft.VisualStudio.TestTools.UnitTesting;

  [TestClass]
  public class ProblemTests {
    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void CountOdds() {
      RandomWalk(new CountOdds());
    }

    [TestMethod]
    [TestProperty("Time", "Medium")]
    [TestCategory("ProblemTest")]
    public void Checksum() {
      RandomWalk(new Checksum());
    }

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void CollatzNumbers() {
    //  RandomWalk(new CollatzNumbers());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void CompareStringLengths() {
    //  RandomWalk(new CompareStringLengths());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void Digits() {
    //  RandomWalk(new Digits());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void DoubleLetters() {
    //  RandomWalk(new DoubleLetters());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void EvenSquares() {
    //  RandomWalk(new EvenSquares());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void ForLoopIndex() {
    //  RandomWalk(new ForLoopIndex());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void Grades() {
    //  RandomWalk(new Grades());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void LastIndexOfZero() {
    //  RandomWalk(new LastIndexOfZero());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void Median() {
    //  RandomWalk(new Median());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void MirrorImage() {
    //  RandomWalk(new MirrorImage());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void NegativeToZero() {
    //  RandomWalk(new NegativeToZero());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void NumberIo() {
    //  RandomWalk(new NumberIo());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void PigLatin() {
    //  RandomWalk(new PigLatin());
    //}

    //[TestMethod]
    //[TestProperty("Time", "Medium")]
    //[TestCategory("ProblemTest")]
    //public void ReplaceSpaceWithNewLine() {
    //  RandomWalk(new ReplaceSpaceWithNewline());
    //}

    private static void RandomWalk(IDataDescriptor descriptor) {
      var maxProgramSizeLimit = 1024;
      var iterations = 100;
      var best = double.MaxValue;
      var globalExecCounter = 0;
      var lockObj = new object();
      var lockCount = new object();
      var random = new FastRandom(1337);

      Expression bestProgram = null;
      var config = new PushConfiguration { EvalPushLimit = 4096 };
      var pool = new PushInterpreterPool(config);

      var instance = new BenchmarkSuiteInstanceProvider();
      var data = instance.LoadData(descriptor);

      Parallel.For(0, iterations, i => {
        var execCounter = 0;
        var program = CodeGenerator.RandomExpandExpression(maxProgramSizeLimit, random);
        var results = new double[data.OriginalTrainingCount];

        using (var interpreter = pool.GetInstance(random)) {
          for (var j = 0; j < data.OriginalTrainingCount; j++) {
            var example = data.Examples[i];

            interpreter.BooleanStack.Push(example.InputBoolean);
            interpreter.IntegerStack.Push(example.InputInt);
            interpreter.FloatStack.Push(example.InputFloat);

            interpreter.Run(program);

            var diff = GetDiff(example.OutputInt, interpreter.IntegerStack) +
                       GetDiff(example.OutputFloat, interpreter.FloatStack) +
                       GetDiff(example.OutputBoolean, interpreter.BooleanStack);

            results[j] = diff;

            execCounter += interpreter.ExecCounter;
            interpreter.Clear();
          }
        }

        lock (lockCount) {
          globalExecCounter += execCounter;
        }

        var avg = results.Average();

        if (avg >= best) return;

        lock (lockObj) {
          if (avg < best) {
            best = avg;
            bestProgram = program;
          }
        }
      });

      var resultsTest = new double[data.OriginalTestCount];
      Parallel.For(data.OriginalTestCount, data.OriginalTestCount, i => {
        using (var interpreter = pool.GetInstance()) {
          var example = data.Examples[i];

          interpreter.BooleanStack.Push(example.InputBoolean);
          interpreter.IntegerStack.Push(example.InputInt);
          interpreter.FloatStack.Push(example.InputFloat);

          interpreter.Run(bestProgram);

          var diff = GetDiff(example.OutputInt, interpreter.IntegerStack) +
                     GetDiff(example.OutputFloat, interpreter.FloatStack) +
                     GetDiff(example.OutputBoolean, interpreter.BooleanStack);

          resultsTest[i] = diff;
        }
      });

      var averageTestResult = resultsTest.Average();

      Console.WriteLine("Best Training: {0}", best);
      Console.WriteLine("Test: {0}", averageTestResult);
      Console.WriteLine("Best trainig program: {0}", bestProgram);
      Console.WriteLine("ExecCounter: {0}", globalExecCounter);
    }

    private static double GetDiff<T>(IReadOnlyList<T> estimated, IStack<T> resultStack)
      where T : IComparable {
      var count = Math.Min(estimated.Count, resultStack.Count);
      var result = resultStack.Peek(count);
      var comparableLength = Math.Min(estimated.Count, result.Length);
      var diff = 0d;

      for (var i = 0; i < comparableLength; i++) {
        diff += Math.Abs(estimated[i].CompareTo(result[i]));
      }

      if (estimated.Count > result.Length) {
        for (var i = comparableLength; i < estimated.Count; i++) {
          diff += Math.Abs(estimated[i].CompareTo(default(T)));
        }
      }

      return diff;
    }
  }
}