source: branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs @ 5119

using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using HeuristicLab.Core;
using HeuristicLab.PluginInfrastructure;
using HeuristicLab.Parameters;
using HeuristicLab.Problems.MetaOptimization;
using HeuristicLab.Data;
using System;
using System.Threading;
using HeuristicLab.Random;
using HeuristicLab.Optimization;
using HeuristicLab.Common;
using System.IO;
using HeuristicLab.Problems.TestFunctions;
using System.Text;
using HeuristicLab.Selection;
using HeuristicLab.Algorithms.EvolutionStrategy;
using HeuristicLab.PluginInfrastructure.Manager;

namespace HeuristicLab.MetaOptimization.Test {
  class Program {
    private static int metaAlgorithmPopulationSize = 50;
    private static int metaAlgorithmMaxGenerations = 30;
    private static int metaProblemRepetitions = 6;

    private static int baseAlgorithmMaxGenerations = 250;

    static void Main(string[] args) {
      //TestShorten();

      //TestIntSampling();
      //TestDoubleSampling();
      //TestTypeDiscovery();
      //TestOperators();
      
      GeneticAlgorithm baseLevelAlgorithm = new GeneticAlgorithm();

      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
      metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
      //EvolutionStrategy metaLevelAlgorithm = GetMetaES(metaOptimizationProblem);

      IValueConfiguration algorithmVc = SetupGAAlgorithm(baseLevelAlgorithm, metaOptimizationProblem);

      //Console.WriteLine("Press enter to start");
      //Console.ReadLine();
      //TestConfiguration(algorithmVc, baseLevelAlgorithm);
      
      //Console.WriteLine("Press enter to start");
      //Console.ReadLine();
      TestOptimization(metaLevelAlgorithm);

      //TestMemoryLeak(metaLevelAlgorithm);

      Console.ReadLine();
    }

    private static void TestOperators() {
      IRandom random = new MersenneTwister();
      ParameterConfigurationManipulator manip = new ParameterConfigurationManipulator();

      manip.IntValueManipulatorParameter.ActualValue = new UniformIntValueManipulator();
      manip.DoubleValueManipulatorParameter.ActualValue = new NormalDoubleValueManipulator();
      
      var doubleRange = new DoubleValueRange(new DoubleValue(0), new DoubleValue(100), new DoubleValue(0.1));
      using (var sw = new StreamWriter("out-DoubleValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new DoubleValue(50);
          NormalDoubleValueManipulator.ApplyStatic(random, val, doubleRange);
          
          sw.WriteLine(val);
        }
      }

      var percentRange = new PercentValueRange(new PercentValue(0), new PercentValue(1), new PercentValue(0.001));
      using (var sw = new StreamWriter("out-PercentValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new PercentValue(0.5);
          NormalDoubleValueManipulator.ApplyStatic(random, val, percentRange.AsDoubleValueRange());
          sw.WriteLine(val);
        }
      }

      var intRange = new IntValueRange(new IntValue(0), new IntValue(100), new IntValue(1));
      using (var sw = new StreamWriter("out-IntValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new IntValue(50);
          UniformIntValueManipulator.ApplyStatic(random, val, intRange);
          sw.WriteLine(val);
        }
      }

      Console.ReadLine();
    }

    private static void TestTypeDiscovery() {
      PluginLoader.pluginAssemblies.Any();
      
      var items = ApplicationManager.Manager.GetInstances(typeof(DoubleArray)).ToArray();

      foreach (var item in items) {
        Console.WriteLine(item.ToString());
      }
    }

    private static void TestMemoryLeak(GeneticAlgorithm metaLevelAlgorithm) {
      IValueConfiguration algorithmVc = ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).AlgorithmParameterConfiguration;

      Console.WriteLine("Starting Memory Test...");
      Console.ReadLine();

      var clones = new List<object>();
      for (int i = 0; i < 1000; i++) {
        var clone = algorithmVc.Clone();
        clones.Add(clone);
      }

      Console.WriteLine("Finished. Now GC...");
      Console.ReadLine();

      GC.Collect();

      Console.WriteLine("Finished!");
      Console.ReadLine();
    }

    private static GeneticAlgorithm GetMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
      GeneticAlgorithm metaLevelAlgorithm = new GeneticAlgorithm();
      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;

      metaLevelAlgorithm.Problem = metaOptimizationProblem;
      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();

      metaLevelAlgorithm.Mutator = new ParameterConfigurationManipulator();
      metaLevelAlgorithm.MutationProbability.Value = 0.15;

      return metaLevelAlgorithm;
    }

    private static EvolutionStrategy GetMetaES(MetaOptimizationProblem metaOptimizationProblem) {
      EvolutionStrategy metaLevelAlgorithm = new EvolutionStrategy();
      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;

      metaLevelAlgorithm.Problem = metaOptimizationProblem;
      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();

      metaLevelAlgorithm.Mutator = new ParameterConfigurationManipulator();
      //metaLevelAlgorithm.MutationProbability.Value = 0.15;

      return metaLevelAlgorithm;
    }

    private static IValueConfiguration SetupGAAlgorithm(GeneticAlgorithm baseLevelAlgorithm, MetaOptimizationProblem metaOptimizationProblem) {
      baseLevelAlgorithm.Problem = new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem();
      baseLevelAlgorithm.MaximumGenerations.Value = baseAlgorithmMaxGenerations;

      metaOptimizationProblem.Algorithm = baseLevelAlgorithm;
      IValueConfiguration algorithmVc = metaOptimizationProblem.AlgorithmParameterConfiguration;

      metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
        Evaluator = new GriewankEvaluator(),
        ProblemSize = new IntValue(500)
      });
      metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
        Evaluator = new GriewankEvaluator(),
        ProblemSize = new IntValue(1000)
      });

      ConfigurePopulationSize(algorithmVc);
      ConfigureMutationRate(algorithmVc);
      ConfigureMutationOperator(algorithmVc);
      ConfigureElites(algorithmVc);
      ConfigureSelectionOperator(algorithmVc);
      return algorithmVc;
    }

    private static void TestConfiguration(IValueConfiguration algorithmVc, GeneticAlgorithm baseLevelAlgorithm) {
      IRandom rand = new FastRandom(0);
      // set random values
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
        GeneticAlgorithm newAlg = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        clonedVc.Randomize(rand);
        clonedVc.Parameterize(newAlg);
        Console.WriteLine(string.Format("PopSize: original: {0}, randomized: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, randomized: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        //Console.WriteLine(string.Format("GrSi: original: {0}, randomized: {1}", "?", ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      algorithmVc.Randomize(rand);
      algorithmVc.Parameterize(baseLevelAlgorithm);
      // mutate
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
        GeneticAlgorithm newAlg = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        //clonedVc.Mutate(rand);
        
        //.Apply(rand, clonedVc); todo
        clonedVc.Parameterize(newAlg);
        Console.WriteLine(string.Format("PopSize: original: {0}, mutated: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, mutated: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        //Console.WriteLine(string.Format("GrSi: original: {0}, mutated: {1}", ((TournamentSelector)baseLevelAlgorithm.Selector).GroupSizeParameter.Value, ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      // cross
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc1 = (IValueConfiguration)algorithmVc.Clone();
        IValueConfiguration clonedVc2 = (IValueConfiguration)algorithmVc.Clone();

        GeneticAlgorithm first = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        GeneticAlgorithm second = (GeneticAlgorithm)baseLevelAlgorithm.Clone();

        clonedVc1.Randomize(rand);
        clonedVc1.Parameterize(first);

        clonedVc2.Randomize(rand);
        clonedVc2.Parameterize(second);

        var popSizeBefore = first.PopulationSize.Value;
        var mutRateBefore = first.MutationProbability.Value;
        var mutOpBefore = first.Mutator;
        var selOpBefore = first.Selector;
        //var groupSizeBefore = ((TournamentSelector)first.Selector).GroupSizeParameter.Value.Value;

        //clonedVc1.Cross(clonedVc2, rand); todo
        clonedVc1.Parameterize(first);

        Console.WriteLine(string.Format("PopSize: first: {0}, second: {1}, crossed: {2}", popSizeBefore, second.PopulationSize, first.PopulationSize));
        Console.WriteLine(string.Format("MutRate: first: {0}, second: {1}, crossed: {2}", mutRateBefore, second.MutationProbability, first.MutationProbability));
        Console.WriteLine(string.Format("MutOp: first: {0}, second: {1}, crossed: {2}", mutOpBefore, second.Mutator, first.Mutator));
        Console.WriteLine(string.Format("SelOp: first: {0}, second: {1}, crossed: {2}", selOpBefore, second.Selector, first.Selector));
        //Console.WriteLine(string.Format("GrSi: first: {0}, second: {1}, crossed: {2}", groupSizeBefore, ((TournamentSelector)second.Selector).GroupSizeParameter.Value, ((TournamentSelector)first.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }
      Console.WriteLine("=======================");
    }

    private static void ConfigureMutationOperator(IValueConfiguration algorithmVc) {
      var mutationOperator = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Mutator").SingleOrDefault();
      mutationOperator.Optimize = true;

      // uncheck multiMutator to avoid Michalewicz issue
      var multiMutator = mutationOperator.ValueConfigurations.Where(x => x.ActualValue.Value != null && x.ActualValue.Value.ItemName.StartsWith("Multi")).SingleOrDefault();
      if (multiMutator != null) {
        mutationOperator.ValueConfigurations.SetItemCheckedState(multiMutator, false);
      }
    }

    private static void ConfigureSelectionOperator(IValueConfiguration algorithmVc) {
      var selectionOperatorPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Selector").SingleOrDefault();
      selectionOperatorPc.Optimize = true;

      foreach (var vc in selectionOperatorPc.ValueConfigurations) {
        if (vc.ActualValue.ValueDataType == typeof(TournamentSelector)) {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
          vc.Optimize = true;
          ConfigureTournamentGroupSize(vc);
        } else if (vc.ActualValue.ValueDataType == typeof(RandomSelector)) {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
        } else {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
        }
      }
    }

    private static void ConfigureTournamentGroupSize(IValueConfiguration tournamentVc) {
      var groupSizePc = tournamentVc.ParameterConfigurations.Where(x => x.ParameterName == "GroupSize").SingleOrDefault();
      groupSizePc.Optimize = true;

      groupSizePc.ValueConfigurations.First().Optimize = true;
      groupSizePc.ValueConfigurations.First().RangeConstraint.LowerBound = new IntValue(0);
      groupSizePc.ValueConfigurations.First().RangeConstraint.UpperBound = new IntValue(100);
      groupSizePc.ValueConfigurations.First().RangeConstraint.StepSize = new IntValue(1);
    }

    private static void ConfigurePopulationSize(IValueConfiguration algorithmVc) {
      var populationSizePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "PopulationSize").SingleOrDefault();
      populationSizePc.Optimize = true;
      var populationSizeVc = populationSizePc.ValueConfigurations.First();
      populationSizeVc.Optimize = true;
      populationSizeVc.RangeConstraint.LowerBound = new IntValue(20);
      populationSizeVc.RangeConstraint.UpperBound = new IntValue(100);
      populationSizeVc.RangeConstraint.StepSize = new IntValue(1);
    }

    private static void ConfigureMutationRate(IValueConfiguration algorithmVc) {
      var mutationRatePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "MutationProbability").SingleOrDefault();
      mutationRatePc.Optimize = true;
      var mutationRateVc = mutationRatePc.ValueConfigurations.First();
      mutationRateVc.Optimize = true;
      mutationRateVc.RangeConstraint.LowerBound = new PercentValue(0.0);
      mutationRateVc.RangeConstraint.UpperBound = new PercentValue(1.0);
      mutationRateVc.RangeConstraint.StepSize = new PercentValue(0.01);
    }

    private static void ConfigureElites(IValueConfiguration algorithmVc) {
      var elitesPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Elites").SingleOrDefault();
      elitesPc.Optimize = true;
      var elitesVc = elitesPc.ValueConfigurations.First();
      elitesVc.Optimize = true;
      elitesVc.RangeConstraint.LowerBound = new IntValue(0);
      elitesVc.RangeConstraint.UpperBound = new IntValue(20);
      elitesVc.RangeConstraint.StepSize = new IntValue(1);
    }

    private static void TestOptimization(EngineAlgorithm metaLevelAlgorithm) {
      ContentManager.Initialize(new PersistenceContentManager());
      string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Results");
      if (!Directory.Exists(path))
        Directory.CreateDirectory(path);
      string id = DateTime.Now.ToString("yyyy.MM.dd - HH;mm;ss,ffff");
      string resultPath = Path.Combine(path, string.Format("{0} - Result.hl", id));
      string outputPath = Path.Combine(path, string.Format("{0} - Console.txt", id));

      using (var sw = new StreamWriter(outputPath)) {
        sw.AutoFlush = true;

        StringBuilder sb1 = new StringBuilder();
        sb1.AppendLine(string.Format("Meta.PopulationSize: {0}", metaAlgorithmPopulationSize));
        sb1.AppendLine(string.Format("Meta.MaxGenerations: {0}", metaAlgorithmMaxGenerations));
        sb1.AppendLine(string.Format("Meta.Repetitions   : {0}", metaProblemRepetitions));
        sb1.AppendLine(string.Format("Base.MaxGenerations: {0}", baseAlgorithmMaxGenerations));
        sw.WriteLine(sb1.ToString());
        Console.WriteLine(sb1.ToString());

        metaLevelAlgorithm.Start();
        int i = 0;
        int currentGeneration = -1;
        do {
          Thread.Sleep(500);
          if (metaLevelAlgorithm.Results.ContainsKey("Generations") && ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value != currentGeneration) {
            while (metaLevelAlgorithm.Results.Count < 3) Thread.Sleep(100);
            StringBuilder sb = new StringBuilder();
            sb.AppendLine(DateTime.Now.ToLongTimeString());
            sb.AppendLine("=================================");

            sb.AppendLine(metaLevelAlgorithm.ExecutionState.ToString());
            foreach (var result in metaLevelAlgorithm.Results) {
              sb.AppendLine(result.ToString());
              if (result.Name == "Population") {
                RunCollection rc = (RunCollection)result.Value;
                var orderedRuns = rc.OrderBy(x => x.Results["RunsAverageQuality"]);

                sb.AppendLine("Qual.  PoSi MutRa Eli SelOp MutOp");
                foreach (IRun run in orderedRuns) {
                  string selector;
                  if (run.Parameters["Selector"] is TournamentSelector) {
                    selector = string.Format("{0} ({1})", run.Parameters["Selector"].ToString(), ((TournamentSelector)run.Parameters["Selector"]).GroupSizeParameter.Value.ToString());
                  } else {
                    selector = string.Format("{0}", run.Parameters["Selector"].ToString());
                  }

                  sb.AppendLine(string.Format("{0} {1} {2} {3} {4} {5}",
                    ((DoubleValue)run.Results["RunsAverageQuality"]).Value.ToString("#0.00").PadLeft(7, ' '),
                    ((IntValue)run.Parameters["PopulationSize"]).Value.ToString().PadLeft(3, ' ').PadRight(3, ' '),
                    ((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.00").PadLeft(5, ' '),
                    ((IntValue)run.Parameters["Elites"]).Value.ToString().PadLeft(3, ' '),
                    Shorten(selector, 20).PadRight(20, ' '),
                    run.Parameters.ContainsKey("Mutator") ? run.Parameters["Mutator"].ToString() : "null"));
                }
              }
            } // foreach
            Console.Clear();
            Console.WriteLine(sb.ToString());
            sw.WriteLine(sb.ToString());
            currentGeneration = ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value;
          } // if
          if (i % 30 == 0) GC.Collect();
          i++;
        } while (metaLevelAlgorithm.ExecutionState != ExecutionState.Stopped);
      }

      Console.WriteLine();
      Console.WriteLine("Storing...");

      ContentManager.Save((IStorableContent)metaLevelAlgorithm, resultPath, true);
      Console.WriteLine("Finished");
    }

    private static void TestShorten() {
      int n = 8;
      Console.WriteLine(Shorten("1", n));
      Console.WriteLine(Shorten("12", n));
      Console.WriteLine(Shorten("123", n));
      Console.WriteLine(Shorten("1234", n));
      Console.WriteLine(Shorten("12345", n));
      Console.WriteLine(Shorten("123456", n));
      Console.WriteLine(Shorten("1234567", n));
      Console.WriteLine(Shorten("12345678", n));
      Console.WriteLine(Shorten("123456789", n));
      Console.WriteLine(Shorten("1234567890", n));
      Console.WriteLine(Shorten("12345678901", n));
    }

    private static string Shorten(string s, int n) {
      string placeholder = "..";
      if (s.Length <= n) return s;
      int len = n / 2 - placeholder.Length / 2;
      string start = s.Substring(0, len);
      string end = s.Substring(s.Length - len, len);
      return start + placeholder + end;
    }

    private static void TestIntSampling() {
      System.Random rand = new System.Random();
      int lower = 10;
      int upper = 20;
      int stepsize = 7;
      for (int i = 0; i < 100; i++) {
        int val;
        do {
          val = rand.Next(lower / stepsize, upper / stepsize + 1) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static void TestDoubleSampling() {
      System.Random rand = new System.Random();
      double lower = 2;
      double upper = 3;
      double stepsize = 0.6;
      for (int i = 0; i < 100; i++) {
        double val;
        do {
          val = Math.Round((rand.NextDouble() * (upper - lower) + lower) / stepsize, 0) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static IEnumerable<IItem> GetValidValues(IValueParameter valueParameter) {
      return ApplicationManager.Manager.GetInstances(valueParameter.DataType).Select(x => (IItem)x).OrderBy(x => x.ItemName);
    }
  }
}