source: branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs @ 5023

using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using HeuristicLab.Core;
using HeuristicLab.PluginInfrastructure;
using HeuristicLab.Parameters;
using HeuristicLab.Problems.MetaOptimization;
using HeuristicLab.Data;
using System;
using System.Threading;
using HeuristicLab.Random;
using HeuristicLab.Optimization;
using HeuristicLab.Common;
using System.IO;
using HeuristicLab.Problems.TestFunctions;
using System.Text;
using HeuristicLab.Selection;

namespace HeuristicLab.MetaOptimization.Test {
  class Program {
    private static int metaAlgorithmPopulationSize = 40;
    private static int metaAlgorithmMaxGenerations = 30;

    private static int baseAlgorithmMaxGenerations = 100;

    static void Main(string[] args) {
      //TestIntSampling();
      //TestDoubleSampling();

      GeneticAlgorithm baseLevelAlgorithm = new GeneticAlgorithm();
      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
      GeneticAlgorithm metaLevelAlgorithm = GetMetaAlgorithm(metaOptimizationProblem);

      IValueConfiguration algorithmVc = SetupAlgorithm(baseLevelAlgorithm, metaOptimizationProblem);

      //Console.WriteLine("Press enter to start");
      //Console.ReadLine();
      //TestConfiguration(algorithmVc, baseLevelAlgorithm);

      //Console.WriteLine("Press enter to start");
      //Console.ReadLine();
      TestOptimization(metaLevelAlgorithm);

      //TestMemoryLeak(metaLevelAlgorithm);

      Console.ReadLine();
    }

    private static void TestMemoryLeak(GeneticAlgorithm metaLevelAlgorithm) {
      IValueConfiguration algorithmVc = ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).AlgorithmParameterConfiguration;

      Console.WriteLine("Starting Memory Test...");
      Console.ReadLine();

      var clones = new List<object>();
      for (int i = 0; i < 1000; i++) {
        var clone = algorithmVc.Clone();
        clones.Add(clone);
      }

      Console.WriteLine("Finished. Now GC...");
      Console.ReadLine();

      GC.Collect();

      Console.WriteLine("Finished!");
      Console.ReadLine();
    }

    private static GeneticAlgorithm GetMetaAlgorithm(MetaOptimizationProblem metaOptimizationProblem) {
      GeneticAlgorithm metaLevelAlgorithm = new GeneticAlgorithm();
      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;

      metaLevelAlgorithm.Problem = metaOptimizationProblem;
      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();

      metaLevelAlgorithm.Mutator = new ParameterConfigurationManipulator();
      metaLevelAlgorithm.MutationProbability.Value = 0.15;

      return metaLevelAlgorithm;
    }

    private static IValueConfiguration SetupAlgorithm(GeneticAlgorithm baseLevelAlgorithm, MetaOptimizationProblem metaOptimizationProblem) {
      baseLevelAlgorithm.Problem = new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() { 
        Evaluator = new GriewankEvaluator(),
        ProblemSize = new IntValue(1000) 
      };
      baseLevelAlgorithm.MaximumGenerations.Value = baseAlgorithmMaxGenerations;

      metaOptimizationProblem.Algorithm = baseLevelAlgorithm;
      IValueConfiguration algorithmVc = metaOptimizationProblem.AlgorithmParameterConfiguration;
      metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem());

      ConfigurePopulationSize(algorithmVc);
      ConfigureMutationRate(algorithmVc);
      ConfigureMutationOperator(algorithmVc);
      ConfigureElites(algorithmVc);
      ConfigureSelectionOperator(algorithmVc);
      return algorithmVc;
    }

    private static void TestConfiguration(IValueConfiguration algorithmVc, GeneticAlgorithm baseLevelAlgorithm) {
      IRandom rand = new FastRandom(0);
      // set random values
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
        clonedVc.Randomize(rand);
        clonedVc.Parameterize((GeneticAlgorithm)clonedVc.ActualValue.Value);
        GeneticAlgorithm newAlg = (GeneticAlgorithm)clonedVc.ActualValue.Value;
        Console.WriteLine(string.Format("PopSize: original: {0}, randomized: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, randomized: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        Console.WriteLine(string.Format("GrSi: original: {0}, randomized: {1}", "?", ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      algorithmVc.Randomize(rand);
      algorithmVc.Parameterize(baseLevelAlgorithm);
      // mutate
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
        clonedVc.Mutate(rand);
        clonedVc.Parameterize((GeneticAlgorithm)clonedVc.ActualValue.Value);
        GeneticAlgorithm newAlg = (GeneticAlgorithm)clonedVc.ActualValue.Value;
        Console.WriteLine(string.Format("PopSize: original: {0}, mutated: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, mutated: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        Console.WriteLine(string.Format("GrSi: original: {0}, mutated: {1}", ((TournamentSelector)baseLevelAlgorithm.Selector).GroupSizeParameter.Value, ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      // cross
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc1 = (IValueConfiguration)algorithmVc.Clone();
        IValueConfiguration clonedVc2 = (IValueConfiguration)algorithmVc.Clone();

        GeneticAlgorithm first = (GeneticAlgorithm)clonedVc1.ActualValue.Value.Clone();
        GeneticAlgorithm second = (GeneticAlgorithm)clonedVc2.ActualValue.Value.Clone();

        clonedVc1.Randomize(rand);
        clonedVc1.Parameterize(first);

        clonedVc2.Randomize(rand);
        clonedVc2.Parameterize(second);

        var popSizeBefore = first.PopulationSize.Value;
        var mutRateBefore = first.MutationProbability.Value;
        var mutOpBefore = first.Mutator;
        var selOpBefore = first.Selector;
        var groupSizeBefore = ((TournamentSelector)first.Selector).GroupSizeParameter.Value.Value;

        clonedVc1.Cross(clonedVc2, rand);
        clonedVc1.Parameterize(first);

        Console.WriteLine(string.Format("PopSize: first: {0}, second: {1}, crossed: {2}", popSizeBefore, second.PopulationSize, first.PopulationSize));
        Console.WriteLine(string.Format("MutRate: first: {0}, second: {1}, crossed: {2}", mutRateBefore, second.MutationProbability, first.MutationProbability));
        Console.WriteLine(string.Format("MutOp: first: {0}, second: {1}, crossed: {2}", mutOpBefore, second.Mutator, first.Mutator));
        Console.WriteLine(string.Format("SelOp: first: {0}, second: {1}, crossed: {2}", selOpBefore, second.Selector, first.Selector));
        Console.WriteLine(string.Format("GrSi: first: {0}, second: {1}, crossed: {2}", groupSizeBefore, ((TournamentSelector)second.Selector).GroupSizeParameter.Value, ((TournamentSelector)first.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }
      Console.WriteLine("=======================");
    }

    private static void ConfigureMutationOperator(IValueConfiguration algorithmVc) {
      var mutationOperator = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Mutator").SingleOrDefault();
      mutationOperator.Optimize = true;

      // uncheck multiMutator to avoid Michalewicz issue
      var multiMutator = mutationOperator.ValueConfigurations.Where(x => x.ActualValue.Value.ItemName.StartsWith("Multi")).SingleOrDefault();
      if (multiMutator != null) {
        mutationOperator.ValueConfigurations.SetItemCheckedState(multiMutator, false);
      }
    }

    private static void ConfigureSelectionOperator(IValueConfiguration algorithmVc) {
      var selectionOperatorPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Selector").SingleOrDefault();
      selectionOperatorPc.Optimize = true;

      foreach (var vc in selectionOperatorPc.ValueConfigurations) {
        if (vc.ActualValue.ValueDataType == typeof(TournamentSelector)) {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
          vc.Optimize = true;
          ConfigureTournamentGroupSize(vc);
        } else {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, false);
        }
      }
    }

    private static void ConfigureTournamentGroupSize(IValueConfiguration tournamentVc) {
      var groupSizePc = tournamentVc.ParameterConfigurations.Where(x => x.ParameterName == "GroupSize").SingleOrDefault();
      groupSizePc.Optimize = true;

      groupSizePc.ValueConfigurations.First().Optimize = true;
      groupSizePc.ValueConfigurations.First().RangeConstraint.LowerBound = new IntValue(0);
      groupSizePc.ValueConfigurations.First().RangeConstraint.UpperBound = new IntValue(100);
      groupSizePc.ValueConfigurations.First().RangeConstraint.StepSize = new IntValue(1);
    }

    private static void ConfigurePopulationSize(IValueConfiguration algorithmVc) {
      var populationSizePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "PopulationSize").SingleOrDefault();
      populationSizePc.Optimize = true;
      var populationSizeVc = populationSizePc.ValueConfigurations.First();
      populationSizeVc.Optimize = true;
      populationSizeVc.RangeConstraint.LowerBound = new IntValue(20);
      populationSizeVc.RangeConstraint.UpperBound = new IntValue(100);
      populationSizeVc.RangeConstraint.StepSize = new IntValue(1);
    }

    private static void ConfigureMutationRate(IValueConfiguration algorithmVc) {
      var mutationRatePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "MutationProbability").SingleOrDefault();
      mutationRatePc.Optimize = true;
      var mutationRateVc = mutationRatePc.ValueConfigurations.First();
      mutationRateVc.Optimize = true;
      mutationRateVc.RangeConstraint.LowerBound = new PercentValue(0.0);
      mutationRateVc.RangeConstraint.UpperBound = new PercentValue(1.0);
      mutationRateVc.RangeConstraint.StepSize = new PercentValue(0.01);
    }

    private static void ConfigureElites(IValueConfiguration algorithmVc) {
      var elitesPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Elites").SingleOrDefault();
      elitesPc.Optimize = true;
      var elitesVc = elitesPc.ValueConfigurations.First();
      elitesVc.Optimize = true;
      elitesVc.RangeConstraint.LowerBound = new IntValue(0);
      elitesVc.RangeConstraint.UpperBound = new IntValue(20);
      elitesVc.RangeConstraint.StepSize = new IntValue(1);
    }

    private static void TestOptimization(GeneticAlgorithm metaLevelAlgorithm) {
      ContentManager.Initialize(new PersistenceContentManager());
      string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Results");
      if(!Directory.Exists(path))
        Directory.CreateDirectory(path);
      string id = DateTime.Now.ToString("MM.dd.yy - HH;mm;ss,ffff");
      string resultPath = Path.Combine(path, string.Format("Test - {0} - Result.hl", id));
      string outputPath = Path.Combine(path, string.Format("Test - {0} - Console.txt", id));

      using (var sw = new StreamWriter(outputPath)) {
        metaLevelAlgorithm.Start();
        int i = 0;
        int currentGeneration = -1;
        do {
          Thread.Sleep(500);
          try {
            if (metaLevelAlgorithm.Results.ContainsKey("Generations") && ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value != currentGeneration) {
              StringBuilder sb = new StringBuilder();
              sb.AppendLine(DateTime.Now.ToLongTimeString());
              sb.AppendLine("=================================");

              foreach (var result in metaLevelAlgorithm.Results) {
                sb.AppendLine(result.ToString());
                if (result.Name == "Population") {
                  RunCollection rc = (RunCollection)result.Value;
                  var orderedRuns = rc.OrderBy(x => x.Results["BestQuality"]);
                  
                  sb.AppendLine("Qality PoSi MutRa Eli GrSi MutOp");
                  foreach (IRun run in orderedRuns) {
                    sb.AppendLine(string.Format("{0} {1} {2} {3} {4} {5}",
                      ((DoubleValue)run.Results["BestQuality"]).Value.ToString("#0.00").PadLeft(7, ' '),
                      ((IntValue)run.Parameters["PopulationSize"]).Value.ToString().PadLeft(3, ' ').PadRight(3, ' '),
                      ((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.00").PadLeft(5, ' '),
                      ((IntValue)run.Parameters["Elites"]).Value.ToString().PadLeft(3, ' '),
                      ((TournamentSelector)run.Parameters["Selector"]).GroupSizeParameter.Value.ToString().PadLeft(4, ' '),
                      run.Parameters["Mutator"]));
                  }
                }
              } // foreach
              Console.Clear();
              Console.WriteLine(sb.ToString());
              sw.WriteLine(sb.ToString());
              sw.Flush();
              currentGeneration = ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value;
            } // if
            if (i % 30 == 0) GC.Collect();
            i++;
          }
          catch { }
        } while (metaLevelAlgorithm.ExecutionState != ExecutionState.Stopped);
      }

      Console.WriteLine();
      Console.WriteLine("Storing...");

      ContentManager.Save(metaLevelAlgorithm, resultPath, true);
      Console.WriteLine("Finished");
    }

    private static void TestIntSampling() {
      System.Random rand = new System.Random();
      int lower = 10;
      int upper = 20;
      int stepsize = 7;
      for (int i = 0; i < 100; i++) {
        int val;
        do {
          val = rand.Next(lower / stepsize, upper / stepsize + 1) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static void TestDoubleSampling() {
      System.Random rand = new System.Random();
      double lower = 2;
      double upper = 3;
      double stepsize = 0.6;
      for (int i = 0; i < 100; i++) {
        double val;
        do {
          val = Math.Round((rand.NextDouble() * (upper - lower) + lower) / stepsize, 0) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static IEnumerable<IItem> GetValidValues(IValueParameter valueParameter) {
      return ApplicationManager.Manager.GetInstances(valueParameter.DataType).Select(x => (IItem)x).OrderBy(x => x.ItemName);
    }
  }
}