source: branches/2886_SymRegGrammarEnumeration/Test/AlgorithmPerformanceTest.cs @ 16175

using System;
using System.Linq;
using System.Threading;
using HeuristicLab.Algorithms.DataAnalysis.SymRegGrammarEnumeration;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Problems.DataAnalysis;
using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
using HeuristicLab.Problems.Instances.DataAnalysis;
using HeuristicLab.Random;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Test {
  [TestClass]
  public class AlgorithmPerformanceTest {
    private const int Seed = 1234;
    private IRandom rand;

    private const double SuccessThreshold = 0.9999999;

    private GrammarEnumerationAlgorithm alg;
    private RegressionProblem problem;

    [TestInitialize]
    public void InitTest() {
      rand = new FastRandom(Seed);
      alg = new GrammarEnumerationAlgorithm();
      //var provider = new PennMLRegressionInstanceProvider();
      //var id = provider.GetDataDescriptors().Single(x => x.Name == "1027_ESL"); // something not randomly generated
      //problem = new RegressionProblem { ProblemData = provider.LoadData(id) };
      problem = new RegressionProblem { ProblemData = new KeijzerFunctionEleven(Seed).GenerateRegressionData() };
      alg.Problem = problem;
      alg.GuiUpdateInterval = int.MaxValue;
      alg.SearchDataStructure = StorageType.Stack;
      alg.SearchDataStructureSize = (int)1e5;

      alg.Stopped += (sender, args) => mutex.Set();
      alg.Started += (sender, args) => mutex.Reset();
    }

    private readonly ManualResetEvent mutex = new ManualResetEvent(false);

    [TestMethod]
    [TestProperty("Performance", "Phrase expansion speed for 2 variables.")]
    public void AlgorithmPerformance() {
      int reps = 5;
      foreach (var st in Enum.GetValues(typeof(StorageType)).Cast<StorageType>()) {
        alg.SearchDataStructure = st;
        double avgExpansionSpeed = 0;
        for (int i = 0; i < reps; ++i) {
          alg.Prepare();
          alg.Start();
          mutex.WaitOne();
          var expansionSpeed = (IntValue)alg.Results["Expansions per second"].Value;
          avgExpansionSpeed += expansionSpeed.Value;
        }

        var distinct = (IntValue)alg.Results["Distinct Sentences"].Value;
        var sentences = (IntValue)alg.Results["Generated Sentences"].Value;
        var expansions = (IntValue)alg.Results["Phrase Expansions"].Value;
        var archived = (IntValue)alg.Results["Generated/Archived Phrases"].Value;
        var overwritten = (IntValue)alg.Results["Sentences overwritten"].Value;
        var best = (SymbolicRegressionSolution)alg.Results["Best solution (Training)"].Value;
        Console.WriteLine(st);
        Console.WriteLine($"Generated/archived phrases: {archived.Value}");
        Console.WriteLine($"Expansions per second: {avgExpansionSpeed / reps}");
        Console.WriteLine($"Generated sentences: {sentences.Value}");
        Console.WriteLine($"Distinct sentences: {distinct.Value}");
        Console.WriteLine($"Overwritten sentences: {overwritten.Value}");
        Console.WriteLine($"Train R2: {best.TrainingRSquared}");
        Console.WriteLine($"Test R2: {best.TestRSquared}");
        Console.WriteLine();
      }
    }
  }
}