Changeset 16500 for branches/2974_Constants_Optimization/UnitTests

Timestamp:

01/04/19 17:37:13 (6 years ago)

Author:

mkommend

Message:

#2974: Added intermediate version of new constants optimization for profiling.

Location:

branches/2974_Constants_Optimization

Files:

• . (modified) (1 prop)
• UnitTests/ConstantsOptimizationTests.cs (modified) (3 diffs)
• UnitTests/PerformanceTest.cs (modified) (1 diff)

branches/2974_Constants_Optimization

@@ -1 +1 @@
 packages
+TestResults

@@ -1 +1 @@
 packages
+TestResults

branches/2974_Constants_Optimization/UnitTests/ConstantsOptimizationTests.cs

@@ -20 +20 @@
 #endregion
 
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using HeuristicLab.Common;
-using HeuristicLab.Core;
-using HeuristicLab.Data;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
-using HeuristicLab.Optimization;
-using HeuristicLab.Parameters;
-using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
-using HeuristicLab.Persistence.Default.Xml;
 using HeuristicLab.Problems.DataAnalysis;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
@@ -38 +28 @@
 
 using HeuristicLab.Problems.Instances.DataAnalysis;
-
-using System;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm;
 
 namespace UnitTests {
@@ -57 +44 @@
 
       var new_optimizedTree = (ISymbolicExpressionTree)tree.Clone();
-      var new_result = ConstantsOptimizationEvaluator.OptimizeConstants(
-        new SymbolicDataAnalysisExpressionTreeLinearInterpreter(),
-        new_optimizedTree, problemData, problemData.TrainingIndices, applyLinearScaling: true, maxIterations: 10);
+      var new_result = LMConstantsOptimizer.OptimizeConstants(new_optimizedTree, problemData, problemData.TrainingIndices, applyLinearScaling: true, maxIterations: 10);
 
       //check R² values
-      Assert.AreEqual(old_result, new_result);
+      //Assert.AreEqual(old_result, new_result);
 
       //check numeric values of constants
       var old_constants = Util.ExtractConstants(old_optimizedTree);
       var new_constants = Util.ExtractConstants(new_optimizedTree);
-      Assert.IsTrue(old_constants.SequenceEqual(new_constants));
-    }
-
-   
+      //Assert.IsTrue(old_constants.SequenceEqual(new_constants));
+
+      for (int i = 0; i < old_constants.Length; i++) {
+        Assert.AreEqual(old_constants[i], new_constants[i], 0.00000001);
+      }
+    }
+
+
 
     [TestMethod]

branches/2974_Constants_Optimization/UnitTests/PerformanceTest.cs

@@ -1 +1 @@
 using System;
-using HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
-using HeuristicLab.Persistence.Default.Xml;
+using HeuristicLab.Problems.DataAnalysis;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
 using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
-using HeuristicLab.Selection;
+using HeuristicLab.Problems.Instances.DataAnalysis;
+using HeuristicLab.Random;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using System.Linq;
-using HeuristicLab.Problems.Instances.DataAnalysis;
-using HeuristicLab.Problems.DataAnalysis;
-using HeuristicLab.Optimization;
-using HeuristicLab.ParallelEngine;
 
 namespace UnitTests {
   [TestClass]
   public class PerformanceTest {
-    private static int seed = 1234;
+    private static readonly int seed = 1234;
+    private static readonly int totalRows = 1000;
+    private static readonly int maxIterations = 10;
+    private static readonly int repetitions = 5;
+    private static readonly int maxTreeSize = 50;
 
-    //TODO remove performance test 
     [TestMethod]
     [TestCategory("Problems.DataAnalysis.Symbolic.Regression")]
     [TestProperty("Time", "long")]
-    public void ConstantsOptimization_Tower_Algorithm() {
-      var algorithm = CreateGpSymbolicRegressionSample();
-      var problem = (SymbolicRegressionSingleObjectiveProblem)algorithm.Problem;
+    public void New_ConstantsOptimization_Tower_Algorithm() {
+      var twister = new MersenneTwister((uint)seed);
+      var problemData = new RegressionRealWorldInstanceProvider().LoadData(new Tower());
+      var rows = Enumerable.Range(0, totalRows);
 
-      algorithm.MaximumGenerations.Value = 1;
-      var evaluator = new ConstantsOptimizationEvaluator();
-      evaluator.ConstantOptimizationIterations.Value = 5;
-      problem.EvaluatorParameter.Value = evaluator;
-
-      algorithm.Prepare();
-      algorithm.Start();
-    }
-
-    private OffspringSelectionGeneticAlgorithm CreateGpSymbolicRegressionSample() {
-      var osga = new OffspringSelectionGeneticAlgorithm();
-      #region Problem Configuration
-      var provider = new RegressionRealWorldInstanceProvider();
-      var problemData =  provider.LoadData(new Tower());
-      var problem = new SymbolicRegressionSingleObjectiveProblem();
-      problem.ProblemData = problemData;
-      problem.Load(problemData);
-      problem.BestKnownQuality.Value = 1.0;
-
-      #region configure grammar
-
-      var grammar = (TypeCoherentExpressionGrammar)problem.SymbolicExpressionTreeGrammar;
+      var grammar = new TypeCoherentExpressionGrammar();
       grammar.ConfigureAsDefaultRegressionGrammar();
 
-      //symbols square, power, squareroot, root, log, exp, sine, cosine, tangent, variable
-      var square = grammar.Symbols.OfType<Square>().Single();
-      var power = grammar.Symbols.OfType<Power>().Single();
-      var squareroot = grammar.Symbols.OfType<SquareRoot>().Single();
-      var root = grammar.Symbols.OfType<Root>().Single();
-      var cube = grammar.Symbols.OfType<Cube>().Single();
-      var cuberoot = grammar.Symbols.OfType<CubeRoot>().Single();
-      var log = grammar.Symbols.OfType<Logarithm>().Single();
-      var exp = grammar.Symbols.OfType<Exponential>().Single();
-      var sine = grammar.Symbols.OfType<Sine>().Single();
-      var cosine = grammar.Symbols.OfType<Cosine>().Single();
-      var tangent = grammar.Symbols.OfType<Tangent>().Single();
-      var variable = grammar.Symbols.OfType<HeuristicLab.Problems.DataAnalysis.Symbolic.Variable>().Single();
-      var powerSymbols = grammar.Symbols.Single(s => s.Name == "Power Functions");
-      powerSymbols.Enabled = true;
+      var trees = CreateRandomTrees(twister, problemData.Dataset, grammar, 1000, 1, maxTreeSize, 0, 0);
+      foreach (SymbolicExpressionTree tree in trees) {
+        InitTree(tree, twister, problemData.AllowedInputVariables.ToList());
+      }
 
-      square.Enabled = true;
-      square.InitialFrequency = 1.0;
-      foreach (var allowed in grammar.GetAllowedChildSymbols(square))
-        grammar.RemoveAllowedChildSymbol(square, allowed);
-      foreach (var allowed in grammar.GetAllowedChildSymbols(square, 0))
-        grammar.RemoveAllowedChildSymbol(square, allowed, 0);
-      grammar.AddAllowedChildSymbol(square, variable);
+      Console.WriteLine("Random tree constants optimization performance of new method:");
 
-      power.Enabled = false;
+      //warm up
+      for (int i = 0; i < trees.Length; i++) {
+        double quality = LMConstantsOptimizer.OptimizeConstants(trees[i], problemData, rows, true, maxIterations);
+      }
 
-      squareroot.Enabled = false;
-      foreach (var allowed in grammar.GetAllowedChildSymbols(squareroot))
-        grammar.RemoveAllowedChildSymbol(squareroot, allowed);
-      foreach (var allowed in grammar.GetAllowedChildSymbols(squareroot, 0))
-        grammar.RemoveAllowedChildSymbol(squareroot, allowed, 0);
-      grammar.AddAllowedChildSymbol(squareroot, variable);
+      Stopwatch watch = new Stopwatch();
+      for (int rep = 0; rep < repetitions; rep++) {
+        watch.Start();
+        for (int i = 0; i < trees.Length; i++) {
+          double quality = LMConstantsOptimizer.OptimizeConstants(trees[i], problemData, rows, true, maxIterations);
+        }
+        watch.Stop();
+        Console.WriteLine("Iteration " + rep + "\t\t" + " Elapsed time: \t" + watch.ElapsedMilliseconds + " ms \t\t" +
+          "Time per tree: " + watch.ElapsedMilliseconds / 1000.0 / trees.Length);
+        watch.Reset();
+      }
+    }
+    [TestMethod]
+    [TestCategory("Problems.DataAnalysis.Symbolic.Regression")]
+    [TestProperty("Time", "long")]
+    public void Old_ConstantsOptimization_Tower_Algorithm() {
+      var twister = new MersenneTwister((uint)seed);
+      var problemData = new RegressionRealWorldInstanceProvider().LoadData(new Tower());
+      var rows = Enumerable.Range(0, totalRows);
 
-      cube.Enabled = false;
-      cuberoot.Enabled = false;
-      root.Enabled = false;
+      var grammar = new TypeCoherentExpressionGrammar();
+      grammar.ConfigureAsDefaultRegressionGrammar();
 
-      log.Enabled = true;
-      log.InitialFrequency = 1.0;
-      foreach (var allowed in grammar.GetAllowedChildSymbols(log))
-        grammar.RemoveAllowedChildSymbol(log, allowed);
-      foreach (var allowed in grammar.GetAllowedChildSymbols(log, 0))
-        grammar.RemoveAllowedChildSymbol(log, allowed, 0);
-      grammar.AddAllowedChildSymbol(log, variable);
+      var trees = CreateRandomTrees(twister, problemData.Dataset, grammar, 1000, 1, maxTreeSize, 0, 0);
+      foreach (SymbolicExpressionTree tree in trees) {
+        InitTree(tree, twister, problemData.AllowedInputVariables.ToList());
+      }
 
-      exp.Enabled = true;
-      exp.InitialFrequency = 1.0;
-      foreach (var allowed in grammar.GetAllowedChildSymbols(exp))
-        grammar.RemoveAllowedChildSymbol(exp, allowed);
-      foreach (var allowed in grammar.GetAllowedChildSymbols(exp, 0))
-        grammar.RemoveAllowedChildSymbol(exp, allowed, 0);
-      grammar.AddAllowedChildSymbol(exp, variable);
+      Console.WriteLine("Random tree constants optimization performance of existing method:");
+      var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
 
-      sine.Enabled = false;
-      foreach (var allowed in grammar.GetAllowedChildSymbols(sine))
-        grammar.RemoveAllowedChildSymbol(sine, allowed);
-      foreach (var allowed in grammar.GetAllowedChildSymbols(sine, 0))
-        grammar.RemoveAllowedChildSymbol(sine, allowed, 0);
-      grammar.AddAllowedChildSymbol(sine, variable);
+      //warm up
+      for (int i = 0; i < trees.Length; i++) {
+        double quality = SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(
+          interpreter, trees[i], problemData, rows, true, maxIterations);
+      }
 
-      cosine.Enabled = false;
-      foreach (var allowed in grammar.GetAllowedChildSymbols(cosine))
-        grammar.RemoveAllowedChildSymbol(cosine, allowed);
-      foreach (var allowed in grammar.GetAllowedChildSymbols(cosine, 0))
-        grammar.RemoveAllowedChildSymbol(cosine, allowed, 0);
-      grammar.AddAllowedChildSymbol(cosine, variable);
-
-      tangent.Enabled = false;
-      foreach (var allowed in grammar.GetAllowedChildSymbols(tangent))
-        grammar.RemoveAllowedChildSymbol(tangent, allowed);
-      foreach (var allowed in grammar.GetAllowedChildSymbols(tangent, 0))
-        grammar.RemoveAllowedChildSymbol(tangent, allowed, 0);
-      grammar.AddAllowedChildSymbol(tangent, variable);
-      #endregion
-
-      problem.SymbolicExpressionTreeGrammar = grammar;
-
-      // configure remaining problem parameters
-      problem.MaximumSymbolicExpressionTreeLength.Value = 50;
-      problem.MaximumSymbolicExpressionTreeDepth.Value = 12;
-      problem.MaximumFunctionDefinitions.Value = 0;
-      problem.MaximumFunctionArguments.Value = 0;
-
-      var evaluator = new SymbolicRegressionConstantOptimizationEvaluator();
-      evaluator.ConstantOptimizationIterations.Value = 5;
-      problem.EvaluatorParameter.Value = evaluator;
-      problem.RelativeNumberOfEvaluatedSamplesParameter.Hidden = true;
-      problem.SolutionCreatorParameter.Hidden = true;
-      #endregion
-
-      #region Algorithm Configuration
-      osga.Problem = problem;
-      osga.Name = "Offspring Selection Genetic Programming - Symbolic Regression";
-      osga.Description = "Genetic programming with strict offspring selection for solving a benchmark regression problem.";
-      ConfigureOsGeneticAlgorithmParameters<GenderSpecificSelector, SubtreeCrossover, MultiSymbolicExpressionTreeManipulator>(osga, 100, 1, 25, 0.2, 50);
-      var mutator = (MultiSymbolicExpressionTreeManipulator)osga.Mutator;
-      mutator.Operators.OfType<FullTreeShaker>().Single().ShakingFactor = 0.1;
-      mutator.Operators.OfType<OnePointShaker>().Single().ShakingFactor = 1.0;
-
-      osga.Analyzer.Operators.SetItemCheckedState(
-        osga.Analyzer.Operators
-          .OfType<SymbolicRegressionSingleObjectiveOverfittingAnalyzer>()
-          .Single(), false);
-      osga.Analyzer.Operators.SetItemCheckedState(
-        osga.Analyzer.Operators
-          .OfType<SymbolicDataAnalysisAlleleFrequencyAnalyzer>()
-          .First(), false);
-
-      osga.ComparisonFactorModifierParameter.Hidden = true;
-      osga.ComparisonFactorLowerBoundParameter.Hidden = true;
-      osga.ComparisonFactorUpperBoundParameter.Hidden = true;
-      osga.OffspringSelectionBeforeMutationParameter.Hidden = true;
-      osga.SuccessRatioParameter.Hidden = true;
-      osga.SelectedParentsParameter.Hidden = true;
-      osga.ElitesParameter.Hidden = true;
-
-      #endregion
-      return osga;
+      Stopwatch watch = new Stopwatch();
+      for (int rep = 0; rep < repetitions; rep++) {
+        watch.Start();
+        for (int i = 0; i < trees.Length; i++) {
+          double quality = SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(
+            interpreter, trees[i], problemData, rows, true, maxIterations);
+        }
+        watch.Stop();
+        Console.WriteLine("Iteration " + rep + "\t\t" + " Elapsed time: \t" + watch.ElapsedMilliseconds + " ms \t\t" +
+          "Time per tree: " + watch.ElapsedMilliseconds / 1000.0 / trees.Length);
+        watch.Reset();
+      }
     }
 
 
-    public static void ConfigureOsGeneticAlgorithmParameters<S, C, M>(OffspringSelectionGeneticAlgorithm ga, int popSize, int elites, int maxGens, double mutationRate = 0.05, double maxSelPres = 100, int tournGroupSize = 0)
-      where S : ISelector
-      where C : ICrossover
-      where M : IManipulator {
-      ga.Elites.Value = elites;
-      ga.MaximumGenerations.Value = maxGens;
-      ga.MutationProbability.Value = mutationRate;
-      ga.PopulationSize.Value = popSize;
-      ga.MaximumSelectionPressure.Value = maxSelPres;
-      ga.Seed.Value = 0;
-      ga.SetSeedRandomly.Value = true;
-      ga.ComparisonFactorLowerBound.Value = 1;
-      ga.ComparisonFactorUpperBound.Value = 1;
 
-      ga.Selector = ga.SelectorParameter.ValidValues
-        .OfType<S>()
-        .First();
+    public static ISymbolicExpressionTree[] CreateRandomTrees(MersenneTwister twister, IDataset dataset, ISymbolicExpressionGrammar grammar, int popSize) {
+      return CreateRandomTrees(twister, dataset, grammar, popSize, 1, 200, 3, 3);
+    }
 
-      ga.Crossover = ga.CrossoverParameter.ValidValues
-        .OfType<C>()
-        .First();
+    public static ISymbolicExpressionTree[] CreateRandomTrees(MersenneTwister twister, IDataset dataset, ISymbolicExpressionGrammar grammar,
+      int popSize, int minSize, int maxSize,
+      int maxFunctionDefinitions, int maxFunctionArguments) {
+      foreach (Variable variableSymbol in grammar.Symbols.OfType<Variable>()) {
+        variableSymbol.VariableNames = dataset.VariableNames;
+      }
+      ISymbolicExpressionTree[] randomTrees = new ISymbolicExpressionTree[popSize];
+      for (int i = 0; i < randomTrees.Length; i++) {
+        randomTrees[i] = ProbabilisticTreeCreator.Create(twister, grammar, maxSize, 10);
+      }
+      return randomTrees;
+    }
 
-      ga.Mutator = ga.MutatorParameter.ValidValues
-        .OfType<M>()
-        .First();
-
-      var tSelector = ga.Selector as TournamentSelector;
-      if (tSelector != null) {
-        tSelector.GroupSizeParameter.Value.Value = tournGroupSize;
+    public static void InitTree(ISymbolicExpressionTree tree, MersenneTwister twister, List<string> varNames) {
+      foreach (var node in tree.IterateNodesPostfix()) {
+        if (node is VariableTreeNode) {
+          var varNode = node as VariableTreeNode;
+          varNode.Weight = twister.NextDouble() * 20.0 - 10.0;
+          varNode.VariableName = varNames[twister.Next(varNames.Count)];
+        } else if (node is ConstantTreeNode) {
+          var constantNode = node as ConstantTreeNode;
+          constantNode.Value = twister.NextDouble() * 20.0 - 10.0;
+        }
       }
-      ga.Engine = new ParallelEngine();
     }
 

@@ -1 +1 @@
 packages
+TestResults