source: branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs @ 5207

using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using HeuristicLab.Algorithms.EvolutionStrategy;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Optimization;
using HeuristicLab.PluginInfrastructure;
using HeuristicLab.Problems.MetaOptimization;
using HeuristicLab.Problems.TestFunctions;
using HeuristicLab.Random;
using HeuristicLab.Selection;
using HeuristicLab.Parameters;
using HeuristicLab.Operators;
using System.Diagnostics;
using HeuristicLab.Encodings.RealVectorEncoding;

namespace HeuristicLab.MetaOptimization.Test {
  class Program {
    private static int metaAlgorithmPopulationSize = 50;
    private static int metaAlgorithmMaxGenerations = 30;
    private static int metaProblemRepetitions = 5;
    private static int baseAlgorithmMaxGenerations = 1000;

    //private static int metaAlgorithmPopulationSize = 10;
    //private static int metaAlgorithmMaxGenerations = 20;
    //private static int metaProblemRepetitions = 3;
    //private static int baseAlgorithmMaxGenerations = 10;

    static void Main(string[] args) {
      //TestTableBuilder();
      //TestShorten();

      //TestIntSampling();
      //TestDoubleSampling(); return;
      //TestTypeDiscovery();
      //TestOperators();
      //TestCombinations();
      //TestCombinations2();
      //TestCombinations3();
      //TestEnumeratorCollectionEnumerator();
      //TestCombinations4();
      //TestAlgorithmPerformanceIssue();

      GeneticAlgorithm baseLevelAlgorithm = new GeneticAlgorithm();

      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
      metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
      //GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
      GeneticAlgorithm metaLevelAlgorithm = GetParallelMetaGA(metaOptimizationProblem);
      //EvolutionStrategy metaLevelAlgorithm = GetMetaES(metaOptimizationProblem);

      IValueConfiguration algorithmVc = SetupGAAlgorithm(baseLevelAlgorithm, metaOptimizationProblem);

      //TestToString(algorithmVc);


      //Console.WriteLine("Press enter to start");
      //Console.ReadLine();
      //TestConfiguration(algorithmVc, baseLevelAlgorithm);

      //Console.WriteLine("Press enter to start");
      //Console.ReadLine();
      TestOptimization(metaLevelAlgorithm);

      //TestMemoryLeak(metaLevelAlgorithm);

      Console.ReadLine();
    }

    private static void TestAlgorithmPerformanceIssue() {
      ContentManager.Initialize(new PersistenceContentManager());
      Queue<TimeSpan> latestExecutionTimes = new Queue<TimeSpan>();
      int size = 10;

      GeneticAlgorithm ga = new GeneticAlgorithm();
      ga.PopulationSize.Value = 3;
      ga.MaximumGenerations.Value = 1;
      ga.Engine = new SequentialEngine.SequentialEngine();

      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
      metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
      ParameterConfigurationTree algorithmVc = SetupGAAlgorithm(ga, metaOptimizationProblem);
      Stopwatch sw = new Stopwatch();

      for (int i = 0; i < 1000; i++) {
        sw.Start();
        GeneticAlgorithm clonedGa = (GeneticAlgorithm)ga.Clone();
        clonedGa.Name = "CLONED GA";
        algorithmVc.Parameterize(clonedGa);
        clonedGa.Prepare(true);
        var executor = new AlgorithmExecutor(clonedGa);
        executor.StartSync();
        sw.Stop();
        latestExecutionTimes.Enqueue(sw.Elapsed);
        Console.WriteLine("{0}: {1} ({2})", i, sw.Elapsed, latestExecutionTimes.Count > size ? TimeSpan.FromMilliseconds(latestExecutionTimes.Average(t => t.TotalMilliseconds)).ToString() : "-");
        if (latestExecutionTimes.Count > size) {
          latestExecutionTimes.Dequeue();
        }
        sw.Reset();
      }
    }

    private static void TestTableBuilder() {
      TableBuilder tb = new TableBuilder("column_1", "col2", "col3");
      tb.AppendRow("1", "humpi", "0.23124");
      tb.AppendRow("2", "sf", "0.23124");
      tb.AppendRow("5", "humpi dampti", "0.224");
      tb.AppendRow("10", "egon asdf", "0.4");
      tb.AppendRow("15", "MichaelizcMultiVfds", "0.23124564");
      Console.WriteLine(tb.ToString());
    }

    private static void TestToInfoString(IValueConfiguration algorithmVc) {
      var random = new MersenneTwister();
      Console.WriteLine(algorithmVc.ParameterInfoString);
      algorithmVc.Randomize(random);
      Console.WriteLine(algorithmVc.ParameterInfoString);
      algorithmVc.Randomize(random);
      Console.WriteLine(algorithmVc.ParameterInfoString);
      algorithmVc.Randomize(random);
    }

    private static void TestCombinations() {
      Console.WriteLine("IntRange 3-18:3");
      IntValueRange intRange = new IntValueRange(new IntValue(3), new IntValue(18), new IntValue(3));
      foreach (var val in intRange.GetCombinations()) {
        Console.WriteLine(val);
      }

      Console.WriteLine("DoubleRange 1.0-2.5:0.5");
      var dblRange = new DoubleValueRange(new DoubleValue(0.7), new DoubleValue(2.8), new DoubleValue(0.5));
      foreach (var val in dblRange.GetCombinations()) {
        Console.WriteLine(val);
      }

      Console.WriteLine("PercentRange 33%-66%:33%");
      var pctRange = new PercentValueRange(new PercentValue(0.32), new PercentValue(0.98), new PercentValue(0.33));
      foreach (var val in pctRange.GetCombinations()) {
        Console.WriteLine(val);
      }
    }

    private static void TestCombinations3() {
      Node root = new Node("root");
      root.ChildNodes.Add(new Node("root.n1"));
      root.ChildNodes.Add(new Node("root.n2"));
      Node n3 = new Node("root.n3");
      n3.ChildNodes.Add(new Node("root.n3.n1"));
      n3.ChildNodes.Add(new Node("root.n3.n2"));
      root.ChildNodes.Add(n3);

      Console.WriteLine(root.ToString());
      Console.WriteLine("--");
      int cnt = 0;
      var enumerator = new NodeEnumerator(root);
      enumerator.Reset();
      while (enumerator.MoveNext()) {
        Console.WriteLine(enumerator.Current.ToString());
        cnt++;
      }
      Console.WriteLine("count: " + cnt);
    }

    private static void TestEnumeratorCollectionEnumerator() {
      IEnumerable<int> list1 = new int[] { 1, 2, 3, 4, 5 };
      IEnumerable<int> list2 = new int[] { 10, 20, 30 };
      IEnumerable<int> list3 = new int[] { 300, 400, 500 };

      var enumerators = new List<IEnumerator>();

      EnumeratorCollectionEnumerator<int> enu = new EnumeratorCollectionEnumerator<int>();
      enu.AddEnumerator(list1.GetEnumerator());
      enu.AddEnumerator(list2.GetEnumerator());
      enu.AddEnumerator(list3.GetEnumerator());
      enu.Reset();
      while (enu.MoveNext()) {
        Console.WriteLine(enu.Current);
      }
    }

    private static void TestCombinations4() {
      GeneticAlgorithm ga = new GeneticAlgorithm();
      ga.Problem = new SingleObjectiveTestFunctionProblem();
      ga.Engine = new SequentialEngine.SequentialEngine();

      ParameterConfigurationTree vc = new ParameterConfigurationTree(ga);

      ConfigurePopulationSize(vc, 20, 100, 20);
      //ConfigureMutationRate(vc, 0.10, 0.60, 0.10);
      //ConfigureMutationOperator(vc);
      ConfigureSelectionOperator(vc, true);

      int count = 0;
      IEnumerator enumerator = new ParameterCombinationsEnumerator(vc);
      enumerator.Reset();
      while (enumerator.MoveNext()) {
        var current = (IValueConfiguration)enumerator.Current;
        count++;
        Console.WriteLine(current.ParameterInfoString);
      }
      Console.WriteLine("You are about to create {0} algorithms.", count);

      Experiment experiment = vc.GenerateExperiment(ga);
      //foreach (var opt in experiment.Optimizers) {
      //  Console.WriteLine(opt.Name);
      //}

      experiment.Prepare();
      experiment.Start();

      while (experiment.ExecutionState != ExecutionState.Stopped) {
        Thread.Sleep(500);
      }
    }

    private static void TestOperators() {
      IRandom random = new MersenneTwister();

      var doubleRange = new DoubleValueRange(new DoubleValue(0), new DoubleValue(100), new DoubleValue(0.1));
      using (var sw = new StreamWriter("out-DoubleValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new DoubleValue(90);
          NormalDoubleValueManipulator.ApplyStatic(random, val, doubleRange);

          sw.WriteLine(val);
        }
      }

      var percentRange = new PercentValueRange(new PercentValue(0), new PercentValue(1), new PercentValue(0.001));
      using (var sw = new StreamWriter("out-PercentValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new PercentValue(0.5);
          NormalDoubleValueManipulator.ApplyStatic(random, val, percentRange.AsDoubleValueRange());
          sw.WriteLine(val);
        }
      }

      var intRange = new IntValueRange(new IntValue(0), new IntValue(100), new IntValue(1));
      using (var sw = new StreamWriter("out-IntValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new IntValue(50);
          UniformIntValueManipulator.ApplyStatic(random, val, intRange);
          sw.WriteLine(val);
        }
      }

      Console.ReadLine();
    }

    private static void TestTypeDiscovery() {
      PluginLoader.pluginAssemblies.Any();

      var items = ApplicationManager.Manager.GetInstances(typeof(DoubleArray)).ToArray();

      foreach (var item in items) {
        Console.WriteLine(item.ToString());
      }
    }

    private static void TestMemoryLeak(GeneticAlgorithm metaLevelAlgorithm) {
      IValueConfiguration algorithmVc = ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).ParameterConfigurationTree;

      Console.WriteLine("Starting Memory Test...");
      Console.ReadLine();

      var clones = new List<object>();
      for (int i = 0; i < 1000; i++) {
        var clone = algorithmVc.Clone();
        clones.Add(clone);
      }

      Console.WriteLine("Finished. Now GC...");
      Console.ReadLine();

      GC.Collect();

      Console.WriteLine("Finished!");
      Console.ReadLine();
    }

    private static GeneticAlgorithm GetMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
      GeneticAlgorithm metaLevelAlgorithm = new GeneticAlgorithm();
      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;

      metaLevelAlgorithm.Problem = metaOptimizationProblem;
      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();

      metaLevelAlgorithm.Mutator = ((OptionalConstrainedValueParameter<IManipulator>)((IAlgorithm)metaLevelAlgorithm).Parameters["Mutator"]).ValidValues.Last();

      metaLevelAlgorithm.MutationProbability.Value = 0.15;

      return metaLevelAlgorithm;
    }

    private static GeneticAlgorithm GetParallelMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);

      UniformSubScopesProcessor uniformSubScopesProcessor =
        (HeuristicLab.Operators.UniformSubScopesProcessor)
        ((HeuristicLab.Operators.AlgorithmOperator)metaLevelAlgorithm.OperatorGraph.Operators.ElementAt(2)).OperatorGraph.Operators.ElementAt(8);
      uniformSubScopesProcessor.Parallel.Value = true;

      metaLevelAlgorithm.Engine = new ParallelEngine.ParallelEngine();

      return metaLevelAlgorithm;
    }

    private static EvolutionStrategy GetMetaES(MetaOptimizationProblem metaOptimizationProblem) {
      EvolutionStrategy metaLevelAlgorithm = new EvolutionStrategy();
      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;

      metaLevelAlgorithm.Problem = metaOptimizationProblem;
      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();

      metaLevelAlgorithm.Mutator = ((OptionalConstrainedValueParameter<IManipulator>)((IAlgorithm)metaLevelAlgorithm).Parameters["Mutator"]).ValidValues.Last();

      return metaLevelAlgorithm;
    }

    private static ParameterConfigurationTree SetupGAAlgorithm(GeneticAlgorithm baseLevelAlgorithm, MetaOptimizationProblem metaOptimizationProblem) {
      baseLevelAlgorithm.Problem = new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem();
      baseLevelAlgorithm.MaximumGenerations.Value = baseAlgorithmMaxGenerations;

      metaOptimizationProblem.Algorithm = baseLevelAlgorithm;
      ParameterConfigurationTree algorithmVc = metaOptimizationProblem.ParameterConfigurationTree;

      metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
        Evaluator = new GriewankEvaluator(),
        ProblemSize = new IntValue(500)
      });
      //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
      //  Evaluator = new GriewankEvaluator(),
      //  ProblemSize = new IntValue(1000)
      //});

      ConfigurePopulationSize(algorithmVc, 0, 100, 1);
      ConfigureMutationRate(algorithmVc, 0.0, 1.0, 0.01);
      ConfigureMutationOperator(algorithmVc);
      ConfigureElites(algorithmVc, 0, 10, 1);
      ConfigureSelectionOperator(algorithmVc, true);
      return algorithmVc;
    }

    private static void TestConfiguration(IValueConfiguration algorithmVc, GeneticAlgorithm baseLevelAlgorithm) {
      IRandom rand = new FastRandom(0);
      // set random values
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
        GeneticAlgorithm newAlg = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        clonedVc.Randomize(rand);
        clonedVc.Parameterize(newAlg);
        Console.WriteLine(string.Format("PopSize: original: {0}, randomized: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, randomized: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        //Console.WriteLine(string.Format("GrSi: original: {0}, randomized: {1}", "?", ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      algorithmVc.Randomize(rand);
      algorithmVc.Parameterize(baseLevelAlgorithm);
      // mutate
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
        GeneticAlgorithm newAlg = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        //clonedVc.Mutate(rand);

        //.Apply(rand, clonedVc); todo
        clonedVc.Parameterize(newAlg);
        Console.WriteLine(string.Format("PopSize: original: {0}, mutated: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, mutated: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        //Console.WriteLine(string.Format("GrSi: original: {0}, mutated: {1}", ((TournamentSelector)baseLevelAlgorithm.Selector).GroupSizeParameter.Value, ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      // cross
      for (int i = 0; i < 10; i++) {
        IValueConfiguration clonedVc1 = (IValueConfiguration)algorithmVc.Clone();
        IValueConfiguration clonedVc2 = (IValueConfiguration)algorithmVc.Clone();

        GeneticAlgorithm first = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        GeneticAlgorithm second = (GeneticAlgorithm)baseLevelAlgorithm.Clone();

        clonedVc1.Randomize(rand);
        clonedVc1.Parameterize(first);

        clonedVc2.Randomize(rand);
        clonedVc2.Parameterize(second);

        var popSizeBefore = first.PopulationSize.Value;
        var mutRateBefore = first.MutationProbability.Value;
        var mutOpBefore = first.Mutator;
        var selOpBefore = first.Selector;
        //var groupSizeBefore = ((TournamentSelector)first.Selector).GroupSizeParameter.Value.Value;

        //clonedVc1.Cross(clonedVc2, rand); todo
        clonedVc1.Parameterize(first);

        Console.WriteLine(string.Format("PopSize: first: {0}, second: {1}, crossed: {2}", popSizeBefore, second.PopulationSize, first.PopulationSize));
        Console.WriteLine(string.Format("MutRate: first: {0}, second: {1}, crossed: {2}", mutRateBefore, second.MutationProbability, first.MutationProbability));
        Console.WriteLine(string.Format("MutOp: first: {0}, second: {1}, crossed: {2}", mutOpBefore, second.Mutator, first.Mutator));
        Console.WriteLine(string.Format("SelOp: first: {0}, second: {1}, crossed: {2}", selOpBefore, second.Selector, first.Selector));
        //Console.WriteLine(string.Format("GrSi: first: {0}, second: {1}, crossed: {2}", groupSizeBefore, ((TournamentSelector)second.Selector).GroupSizeParameter.Value, ((TournamentSelector)first.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }
      Console.WriteLine("=======================");
    }

    private static void ConfigureMutationOperator(IValueConfiguration algorithmVc) {
      var mutationOperator = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Mutator").SingleOrDefault();
      mutationOperator.Optimize = true;

      // uncheck multiMutator to avoid Michalewicz issue
      var multiMutator = mutationOperator.ValueConfigurations.Where(x => x.ActualValue.Value != null && x.ActualValue.Value.ItemName.StartsWith("Multi")).SingleOrDefault();
      if (multiMutator != null) {
        mutationOperator.ValueConfigurations.SetItemCheckedState(multiMutator, false);
      }
    }

    private static void ConfigureSelectionOperator(IValueConfiguration algorithmVc, bool configureTournamenSize) {
      var selectionOperatorPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Selector").SingleOrDefault();
      selectionOperatorPc.Optimize = true;

      foreach (var vc in selectionOperatorPc.ValueConfigurations) {
        if (vc.ActualValue.ValueDataType == typeof(TournamentSelector)) {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
          if (configureTournamenSize) {
            vc.Optimize = true;
            ConfigureTournamentGroupSize(vc);
          }
        } else if (vc.ActualValue.ValueDataType == typeof(RandomSelector)) {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
        } else {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
        }
      }
    }

    private static void ConfigureTournamentGroupSize(IValueConfiguration tournamentVc) {
      var groupSizePc = tournamentVc.ParameterConfigurations.Where(x => x.ParameterName == "GroupSize").SingleOrDefault();
      groupSizePc.Optimize = true;

      groupSizePc.ValueConfigurations.First().Optimize = true;
      groupSizePc.ValueConfigurations.First().RangeConstraint.LowerBound = new IntValue(0);
      groupSizePc.ValueConfigurations.First().RangeConstraint.UpperBound = new IntValue(10);
      groupSizePc.ValueConfigurations.First().RangeConstraint.StepSize = new IntValue(1);
    }

    private static void ConfigurePopulationSize(IValueConfiguration algorithmVc, int lower, int upper, int stepsize) {
      var populationSizePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "PopulationSize").SingleOrDefault();
      populationSizePc.Optimize = true;
      var populationSizeVc = populationSizePc.ValueConfigurations.First();
      populationSizeVc.Optimize = true;
      populationSizeVc.RangeConstraint.LowerBound = new IntValue(lower);
      populationSizeVc.RangeConstraint.UpperBound = new IntValue(upper);
      populationSizeVc.RangeConstraint.StepSize = new IntValue(stepsize);
    }

    private static void ConfigureMutationRate(IValueConfiguration algorithmVc, double lower, double upper, double stepsize) {
      var mutationRatePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "MutationProbability").SingleOrDefault();
      mutationRatePc.Optimize = true;
      var mutationRateVc = mutationRatePc.ValueConfigurations.First();
      mutationRateVc.Optimize = true;
      mutationRateVc.RangeConstraint.LowerBound = new PercentValue(lower);
      mutationRateVc.RangeConstraint.UpperBound = new PercentValue(upper);
      mutationRateVc.RangeConstraint.StepSize = new PercentValue(stepsize);
    }

    private static void ConfigureElites(IValueConfiguration algorithmVc, int from, int to, int stepSize) {
      var elitesPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Elites").SingleOrDefault();
      elitesPc.Optimize = true;
      var elitesVc = elitesPc.ValueConfigurations.First();
      elitesVc.Optimize = true;
      elitesVc.RangeConstraint.LowerBound = new IntValue(from);
      elitesVc.RangeConstraint.UpperBound = new IntValue(to);
      elitesVc.RangeConstraint.StepSize = new IntValue(stepSize);
    }

    private static void TestOptimization(EngineAlgorithm metaLevelAlgorithm) {
      ContentManager.Initialize(new PersistenceContentManager());
      string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Results");
      if (!Directory.Exists(path))
        Directory.CreateDirectory(path);
      string id = DateTime.Now.ToString("yyyy.MM.dd - HH;mm;ss,ffff");
      string resultPath = Path.Combine(path, string.Format("{0} - Result.hl", id));
      string outputPath = Path.Combine(path, string.Format("{0} - Console.txt", id));

      using (var sw = new StreamWriter(outputPath)) {
        sw.AutoFlush = true;

        StringBuilder sb1 = new StringBuilder();
        sb1.AppendFormat("Meta.PopulationSize: {0}\n", metaAlgorithmPopulationSize);
        sb1.AppendFormat("Meta.MaxGenerations: {0}\n", metaAlgorithmMaxGenerations);
        sb1.AppendFormat("Meta.Repetitions   : {0}\n", metaProblemRepetitions);
        sb1.AppendFormat("Meta.MutProb       : {0}\n", ((GeneticAlgorithm)metaLevelAlgorithm).MutationProbability.Value);
        sb1.AppendFormat("Base.MaxGenerations: {0}\n", baseAlgorithmMaxGenerations);
        sb1.AppendLine("Problems:");
        foreach (var prob in ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).Problems) {
          sb1.Append(prob.Name);
          var sotf = prob as SingleObjectiveTestFunctionProblem;
          if (sotf != null) {
            sb1.AppendFormat(" {0}", sotf.ProblemSize.Value);
          }
          sb1.AppendLine();
        }
        sw.WriteLine(sb1.ToString());
        Console.WriteLine(sb1.ToString());
        metaLevelAlgorithm.Stopped += new EventHandler(metaLevelAlgorithm_Stopped);
        metaLevelAlgorithm.Paused += new EventHandler(metaLevelAlgorithm_Paused);
        metaLevelAlgorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(metaLevelAlgorithm_ExceptionOccurred);

        metaLevelAlgorithm.Start();
        int i = 0;
        int currentGeneration = -1;
        do {
          Thread.Sleep(500);
          if (metaLevelAlgorithm.Results.ContainsKey("Generations") && ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value != currentGeneration) {
            while (metaLevelAlgorithm.Results.Count < 3) Thread.Sleep(100);
            StringBuilder sb = new StringBuilder();
            sb.AppendLine(DateTime.Now.ToLongTimeString());
            sb.AppendLine("=================================");

            sb.AppendLine(metaLevelAlgorithm.ExecutionState.ToString());
            var rsClone = (ResultCollection)metaLevelAlgorithm.Results.Clone();
            foreach (var result in rsClone) {
              sb.AppendLine(result.ToString());
              if (result.Name == "Population") {
                RunCollection rc = (RunCollection)result.Value;
                var orderedRuns = rc.OrderBy(x => x.Results["RunsAverageQuality"]);

                TableBuilder tb = new TableBuilder("AvgQual", "AvgET", "PoSi", "MutRa", "Eli", "SelOp", "MutOp", "NrSelSubScopes");
                foreach (IRun run in orderedRuns) {
                  string selector;
                  if (run.Parameters["Selector"] is TournamentSelector) {
                    selector = string.Format("{0} ({1})", run.Parameters["Selector"].ToString(), ((TournamentSelector)run.Parameters["Selector"]).GroupSizeParameter.Value.ToString());
                  } else {
                    selector = string.Format("{0}", run.Parameters["Selector"].ToString());
                  }

                  tb.AppendRow(
                    ((DoubleValue)run.Results["RunsAverageQuality"]).Value.ToString("#0.00"),
                    run.Results["RunsAverageExecutionTime"].ToString(),
                    ((IntValue)run.Parameters["PopulationSize"]).Value.ToString(),
                    ((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.00"),
                    ((IntValue)run.Parameters["Elites"]).Value.ToString(),
                    Shorten(selector, 20),
                    Shorten(run.Parameters.ContainsKey("Mutator") ? run.Parameters["Mutator"].ToString() : "null", 40),
                    ((ISelector)run.Parameters["Selector"]).NumberOfSelectedSubScopesParameter.Value.ToString());
                }
                sb.AppendLine(tb.ToString());
              }
            } // foreach
            //Console.Clear();
            Console.WriteLine(sb.ToString());
            sw.WriteLine(sb.ToString());
            currentGeneration = ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value;
          } // if
          //if (i % 30 == 0) GC.Collect();
          i++;
        } while (metaLevelAlgorithm.ExecutionState != ExecutionState.Stopped);
      }

      Console.WriteLine();
      Console.WriteLine("Storing...");

      ContentManager.Save((IStorableContent)metaLevelAlgorithm, resultPath, true);
      Console.WriteLine("Finished");
    }

    private static void metaLevelAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
      Console.WriteLine("metaLevelAlgorithm_ExceptionOccurred");
    }

    private static void metaLevelAlgorithm_Paused(object sender, EventArgs e) {
      Console.WriteLine("metaLevelAlgorithm_Paused");
    }

    private static void metaLevelAlgorithm_Stopped(object sender, EventArgs e) {
      Console.WriteLine("metaLevelAlgorithm_Stopped");
    }

    private static void TestShorten() {
      int n = 8;
      Console.WriteLine(Shorten("1", n));
      Console.WriteLine(Shorten("12", n));
      Console.WriteLine(Shorten("123", n));
      Console.WriteLine(Shorten("1234", n));
      Console.WriteLine(Shorten("12345", n));
      Console.WriteLine(Shorten("123456", n));
      Console.WriteLine(Shorten("1234567", n));
      Console.WriteLine(Shorten("12345678", n));
      Console.WriteLine(Shorten("123456789", n));
      Console.WriteLine(Shorten("1234567890", n));
      Console.WriteLine(Shorten("12345678901", n));
    }

    private static string Shorten(string s, int n) {
      string placeholder = "..";
      if (s.Length <= n) return s;
      int len = n / 2 - placeholder.Length / 2;
      string start = s.Substring(0, len);
      string end = s.Substring(s.Length - len, len);
      return start + placeholder + end;
    }

    private static void TestIntSampling() {
      System.Random rand = new System.Random();
      int lower = 10;
      int upper = 20;
      int stepsize = 1;
      for (int i = 0; i < 100; i++) {
        int val;
        do {
          val = rand.Next(lower / stepsize, upper / stepsize + 1) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static void TestDoubleSampling() {
      System.Random rand = new System.Random();
      double lower = 2;
      double upper = 3;
      double stepsize = 0.6;
      for (int i = 0; i < 100; i++) {
        double val;
        do {
          val = Math.Round((rand.NextDouble() * (upper - lower) + lower) / stepsize, 0) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static IEnumerable<IItem> GetValidValues(IValueParameter valueParameter) {
      return ApplicationManager.Manager.GetInstances(valueParameter.DataType).Select(x => (IItem)x).OrderBy(x => x.ItemName);
    }
  }

  public class Node {
    public string Name { get; set; }
    public int ActualValue { get; set; }
    public int[] PossibleValues { get; set; }
    public List<Node> ChildNodes { get; set; }

    public Node(string name) {
      this.Name = name;
      PossibleValues = new int[] { 1, 2, 3 };
      ChildNodes = new List<Node>();
    }

    public void Init() {
      this.ActualValue = PossibleValues.First();
      foreach (var child in ChildNodes) {
        child.Init();
      }
    }

    public override string ToString() {
      StringBuilder sb = new StringBuilder();
      sb.Append(string.Format("{0}:{1}", this.Name, this.ActualValue));
      if (this.ChildNodes.Count() > 0) {
        sb.Append(" (");
        var lst = new List<string>();
        foreach (Node child in ChildNodes) {
          lst.Add(child.ToString());
        }
        sb.Append(string.Join(", ", lst.ToArray()));
        sb.Append(")");
      }

      return sb.ToString();
    }
  }

  public class NodeEnumerator : IEnumerator<Node> {
    private Node node;
    private List<IEnumerator> enumerators;

    public NodeEnumerator(Node node) {
      this.node = node;
      this.enumerators = new List<IEnumerator>();
    }

    public Node Current {
      get { return node; }
    }
    object IEnumerator.Current {
      get { return Current; }
    }

    public void Dispose() { }

    public bool MoveNext() {
      int i = 0;
      bool ok = false;
      while (!ok && i < enumerators.Count) {
        if (enumerators[i].MoveNext()) {
          ok = true;
        } else {
          i++;
        }
      }

      if (ok) {
        for (int k = i - 1; k >= 0; k--) {
          enumerators[k].Reset();
          enumerators[k].MoveNext();
        }
      } else {
        return false;
      }

      node.ActualValue = (int)enumerators[0].Current;
      return true;
    }

    public void Reset() {
      enumerators.Clear();
      enumerators.Add(node.PossibleValues.GetEnumerator());
      enumerators[0].Reset();

      foreach (var child in node.ChildNodes) {
        var enumerator = new NodeEnumerator(child);
        enumerator.Reset();
        enumerator.MoveNext();
        enumerators.Add(enumerator);
      }
    }
  }
}