source: branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs @ 5927

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using HeuristicLab.Algorithms.EvolutionStrategy;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Data;
using HeuristicLab.Encodings.RealVectorEncoding;
using HeuristicLab.Hive.ExperimentManager;
using HeuristicLab.Optimization;
using HeuristicLab.Parameters;
using HeuristicLab.PluginInfrastructure;
using HeuristicLab.PluginInfrastructure.Manager;
using HeuristicLab.Problems.DataAnalysis.Symbolic;
using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
using HeuristicLab.Problems.MetaOptimization;
using HeuristicLab.Problems.TestFunctions;
using HeuristicLab.Random;
using HeuristicLab.Selection;

namespace HeuristicLab.MetaOptimization.Test {
  class Program {
    static void Main(string[] args) {
      //PluginLoader.pluginAssemblies.Any();
      PluginManager pm = new PluginManager(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location));
      pm.DiscoverAndCheckPlugins();

      pm.Run(pm.Applications.Where(x => x.Name == "TestApp").SingleOrDefault());
    }
  }

  [Plugin("TestPlugin", "1.0.0.0")]
  [PluginFile("HeuristicLab.MetaOptimization.Test.exe", PluginFileType.Assembly)]
  public class TestPlugin : PluginBase { }

  [Application("TestApp")]
  public class TestApp : ApplicationBase {
    //private static int metaAlgorithmPopulationSize = 50;
    //private static int metaAlgorithmMaxGenerations = 30;
    //private static int metaProblemRepetitions = 5;
    //private static int baseAlgorithmMaxGenerations = 1000;

    private static int metaAlgorithmPopulationSize = 10;
    private static int metaAlgorithmMaxGenerations = 15;
    private static int metaProblemRepetitions = 3;
    private static int baseAlgorithmMaxGenerations = 10;
    private static double mutationProbability = 0.10;

    public override void Run() {
      ContentManager.Initialize(new PersistenceContentManager());

      //TestTableBuilder();
      //TestShorten();

      //TestSimilarities(); return;
      //TestIntSampling();
      //TestDoubleSampling(); return;
      //TestTypeDiscovery();
      //TestOperators();
      //TestCombinations();
      //TestCombinations2();
      //TestCombinations3();
      //TestEnumeratorCollectionEnumerator();
      //TestCombinations4(); return;
      //TestAlgorithmPerformanceIssue(); return;
      //TestWaitAny();
      //TestExecutionTimeUpdateInvervalPerformance();
      //TestMemoryConsumption();
      //TestNormalCrossover();
      //TestItemDictionary();

      //TestSymbolicDataAnalysisGrammar(); return;

      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
      metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
      //GeneticAlgorithm metaLevelAlgorithm = GetParallelMetaGA(metaOptimizationProblem);
      //GeneticAlgorithm metaLevelAlgorithm = GetHiveParallelMetaGA(metaOptimizationProblem);

      //EvolutionStrategy metaLevelAlgorithm = GetMetaES(metaOptimizationProblem);

      var algorithmVc = SetupGAAlgorithm(typeof(GeneticAlgorithm), metaOptimizationProblem);

      //TestToString(algorithmVc);


      //Console.WriteLine("Press enter to start");
      //Console.ReadLine();
      TestConfiguration(algorithmVc, typeof(GeneticAlgorithm), metaOptimizationProblem.Problems.First());

      Console.WriteLine("Press enter to start");
      Console.ReadLine();
      TestOptimization(metaLevelAlgorithm);

      //TestMemoryLeak(metaLevelAlgorithm);

      Console.ReadLine();
    }

    private void TestSymbolicDataAnalysisGrammar() {
      var random = new MersenneTwister();

      var grammar1 = new TypeCoherentExpressionGrammar();
      var grammar2 = new TypeCoherentExpressionGrammar();

      grammar2.Symbols.Single(x => x.Name == "Addition").InitialFrequency = 0.5;

      Console.WriteLine("========== Grammar1: ==========");
      PrintGrammar(grammar1);
      Console.WriteLine("========== Grammar2: ==========");
      PrintGrammar(grammar2);


      RealVector v1 = GetInitialFrequenciesAsRealVector(grammar1);
      RealVector v2 = GetInitialFrequenciesAsRealVector(grammar2);

      for (int i = 0; i < 10; i++) {
        RealVector v3 = DiscreteCrossover.Apply(random, new ItemArray<RealVector>(new List<RealVector> { v1, v2 }));

        var grammar3 = new TypeCoherentExpressionGrammar();
        SetInitialFrequenciesFromRealVector(grammar3, v3);

        Console.WriteLine("\n========== Crossed: ==========");
        PrintGrammar(grammar3);
      }

    }

    private static void PrintGrammar(TypeCoherentExpressionGrammar grammar) {
      foreach (var symbol in grammar.Symbols) {
        Console.WriteLine("{0} ({1})", symbol.ToString(), symbol.InitialFrequency);
      }
    }

    private static RealVector GetInitialFrequenciesAsRealVector(TypeCoherentExpressionGrammar grammar) {
      var vector = new RealVector(grammar.Symbols.Count());
      for (int i = 0; i < grammar.Symbols.Count(); i++) {
        vector[i] = grammar.Symbols.ElementAt(i).InitialFrequency;
      }
      return vector;
    }

    private static void SetInitialFrequenciesFromRealVector(TypeCoherentExpressionGrammar grammar, RealVector vector) {
      for (int i = 0; i < grammar.Symbols.Count(); i++) {
        grammar.Symbols.ElementAt(i).InitialFrequency = vector[i];
      }
    }

    private static void TestSimilarities() {
      Console.WriteLine("\nDoubleRange:");
      var doubleRange = new DoubleValueRange(new DoubleValue(0), new DoubleValue(10), new DoubleValue(1));
      var a = new DoubleValue(5.0);

      for (double d = 0; d < 10; d += 0.1) {
        var similarity = doubleRange.CalculateSimilarity(a, new DoubleValue(d));
        Console.WriteLine("{0}: {1}", d, similarity);
      }

      Console.WriteLine("\nPecentRange:");
      var percentRange = new PercentValueRange(new PercentValue(0), new PercentValue(1), new PercentValue(1));
      var b = new PercentValue(0.05);

      for (double d = 0; d < 1; d += 0.01) {
        var similarity = percentRange.CalculateSimilarity(b, new PercentValue(d));
        Console.WriteLine("{0}: {1}", d, similarity);
      }

      Console.WriteLine("\nIntRange:");
      var intRange = new IntValueRange(new IntValue(50), new IntValue(100), new IntValue(1));
      var c = new IntValue(90);

      for (int i = 0; i < 100; i++) {
        var similarity = intRange.CalculateSimilarity(c, new IntValue(i));
        Console.WriteLine("{0}: {1}", i, similarity);
      }

      Console.WriteLine("\nValueConfigurations:");
      var vc1 = SetupGAAlgorithm(typeof(GeneticAlgorithm), new MetaOptimizationProblem());
      vc1.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "Elites").Optimize = true;
      vc1.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "PopulationSize").Optimize = true;
      vc1.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MutationProbability").Optimize = true;
      vc1.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "Selector").Optimize = true;

      var vc2 = (ParameterConfigurationTree)vc1.Clone();
      Console.WriteLine("Assert(1): {0}", vc1.CalculateSimilarity(vc2));

      ((IntValue)vc2.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "PopulationSize").ValueConfigurations[0].ActualValue.Value).Value = 75;
      Console.WriteLine("{0}", vc1.CalculateSimilarity(vc2));

      ((PercentValue)vc2.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MutationProbability").ValueConfigurations[0].ActualValue.Value).Value = 0.15;
      Console.WriteLine("{0}", vc1.CalculateSimilarity(vc2));

      ((PercentValue)vc2.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MutationProbability").ValueConfigurations[0].ActualValue.Value).Value = 0.25;
      Console.WriteLine("{0}", vc1.CalculateSimilarity(vc2));
      ((PercentValue)vc2.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MutationProbability").ValueConfigurations[0].ActualValue.Value).Value = 0.35;
      Console.WriteLine("{0}", vc1.CalculateSimilarity(vc2));
      ((PercentValue)vc2.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MutationProbability").ValueConfigurations[0].ActualValue.Value).Value = 0.45;
      Console.WriteLine("{0}", vc1.CalculateSimilarity(vc2));
      ((PercentValue)vc2.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MutationProbability").ValueConfigurations[0].ActualValue.Value).Value = 0.55;
      Console.WriteLine("{0}", vc1.CalculateSimilarity(vc2));

      vc2.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "Selector").ActualValueConfigurationIndex = 3;
      Console.WriteLine("{0}", vc1.CalculateSimilarity(vc2));

      var random = new Random.MersenneTwister(0);
      for (int i = 0; i < 10; i++) {
        vc2.Randomize(random);
        Console.WriteLine("Randomized: {0}", vc1.CalculateSimilarity(vc2));
      }
    }

    private static void TestItemDictionary() {
      var dict = new ItemDictionary<StringValue, RunCollection>();
      dict.Add(new StringValue("a"), new RunCollection());
      dict.Add(new StringValue("b"), new RunCollection());
      dict.Add(new StringValue("c"), new RunCollection());

      Console.WriteLine(dict.ContainsKey(new StringValue("a")));
      Console.WriteLine(dict.Count(x => x.Key.Value == "a"));

    }

    private static void TestNormalCrossover() {
      var random = new MersenneTwister();
      double d1 = 0.5;
      double d2 = 0.6;
      var doubleRange = new DoubleValueRange(new DoubleValue(0.0), new DoubleValue(1.0), new DoubleValue(0.01));

      using (var sw = new StreamWriter("normalCrossover-DoubleValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          sw.WriteLine(NormalDoubleValueCrossover.ApplyStatic(random, new DoubleValue(d1), new DoubleValue(d2), doubleRange));
        }
      }

      int i1 = 180;
      int i2 = 160;
      var intRange = new IntValueRange(new IntValue(100), new IntValue(200), new IntValue(1));

      using (var sw = new StreamWriter("normalCrossover-IntValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          sw.WriteLine(NormalIntValueCrossover.ApplyStatic(random, new IntValue(i1), new IntValue(i2), intRange));
        }
      }
    }

    private static void TestMemoryConsumption() {
      Queue<TimeSpan> latestExecutionTimes = new Queue<TimeSpan>();
      GeneticAlgorithm ga = new GeneticAlgorithm();
      ga.PopulationSize.Value = 3;
      ga.MaximumGenerations.Value = 1;
      ga.Engine = new SequentialEngine.SequentialEngine();
      throw new NotImplementedException("TODO: set ga properties correctly");

      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
      metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
      ParameterConfigurationTree algorithmVc = SetupGAAlgorithm(typeof(GeneticAlgorithm), metaOptimizationProblem);
      Stopwatch sw = new Stopwatch();

      var algs = new List<IAlgorithm>();
      for (int i = 0; i < 10000; i++) {
        sw.Start();
        GeneticAlgorithm clonedGa = (GeneticAlgorithm)ga.Clone();
        clonedGa.Name = "CLONED GA";
        algorithmVc.Parameterize(clonedGa);
        algs.Add(clonedGa);
        sw.Reset();
        ContentManager.Save((IStorableContent)metaLevelAlgorithm, "alg_" + i + ".hl", true);
        Console.WriteLine("Cloned alg #{0}", i);
      }
    }

    private static void TestExecutionTimeUpdateInvervalPerformance() {
      TableBuilder tb = new TableBuilder("Tasks", "Interval", "TotalExecutionTime", "AvgExecutionTime", "TimeElapsed", "TotalTimeElapsed", "Speedup", "ExecutionTimeChangedCount", "RealExecutionTimeUpdate(ms)");
      int tasks = 4;
      int repetitions = 3;

      // warmup
      RepeatExecuteParallel(3, 1, 1, tb);
      tb.AppendRow("--", "--", "--", "--", "--", "--", "--", "--", "--");
      RepeatExecuteParallel(repetitions, tasks, 1, tb);
      RepeatExecuteParallel(repetitions, tasks, 2.5, tb);
      RepeatExecuteParallel(repetitions, tasks, 5, tb);
      RepeatExecuteParallel(repetitions, tasks, 10, tb);
      RepeatExecuteParallel(repetitions, tasks, 25, tb);
      RepeatExecuteParallel(repetitions, tasks, 50, tb);
      RepeatExecuteParallel(repetitions, tasks, 100, tb);
      RepeatExecuteParallel(repetitions, tasks, 250, tb);
      RepeatExecuteParallel(repetitions, tasks, 500, tb);
      RepeatExecuteParallel(repetitions, tasks, 1000, tb);
      RepeatExecuteParallel(repetitions, tasks, 2500, tb);
      RepeatExecuteParallel(repetitions, tasks, 5000, tb);

      using (var sw = new StreamWriter("TestExecutionTimeUpdateInvervalPerformance.txt")) {
        sw.Write(tb.ToString());
      }
    }

    private static GeneticAlgorithm CreateGA() {
      GeneticAlgorithm ga = new GeneticAlgorithm();
      ga.Problem = new SingleObjectiveTestFunctionProblem() { ProblemSize = new IntValue(250) };
      ga.Engine = new SequentialEngine.SequentialEngine();
      ga.SetSeedRandomly.Value = false;
      ga.Seed.Value = 0;
      return ga;
    }

    private static void RepeatExecuteParallel(int repetitions, int tasks, double executionTimeUpdateIntervalMs, TableBuilder tb) {
      for (int i = 0; i < repetitions; i++) {
        ExecuteParallel(tasks, executionTimeUpdateIntervalMs, tb);
        Console.Clear();
        Console.WriteLine(tb.ToString());
      }
    }

    private static void ExecuteParallel(int taskCount, double executionTimeUpdateIntervalMs, TableBuilder tb) {
      Task<TimeSpan>[] tasks = new Task<TimeSpan>[taskCount];
      EngineAlgorithm[] algs = new EngineAlgorithm[taskCount];
      for (int i = 0; i < taskCount; i++) {
        GeneticAlgorithm alg = CreateGA();
        //((Engine)alg.Engine).ExecutionTimeUpdateInterval = TimeSpan.FromMilliseconds(executionTimeUpdateIntervalMs);
        algs[i] = alg;
      }
      Console.WriteLine("Creating algs finished.");

      for (int i = 0; i < taskCount; i++) {
        tasks[i] = new Task<TimeSpan>((alg) => {
          Console.WriteLine("Task {0} started.", Task.CurrentId);

          Stopwatch swx = new Stopwatch();
          swx.Start();
          ((EngineAlgorithm)alg).ExecutionTimeChanged += new EventHandler(Program_ExecutionTimeChanged);
          ((EngineAlgorithm)alg).StartSync();
          ((EngineAlgorithm)alg).ExecutionTimeChanged -= new EventHandler(Program_ExecutionTimeChanged);
          swx.Stop();
          Console.WriteLine("Task {0} finished.", Task.CurrentId);
          return swx.Elapsed;
        }, algs[i]);
      }
      Console.WriteLine("Creating tasks finished.");
      counter = 0;
      Stopwatch sw = new Stopwatch();
      sw.Start();
      foreach (var task in tasks) task.Start();
      Task.WaitAll(tasks);
      sw.Stop();

      if (!algs.All(alg => alg.ExecutionState == ExecutionState.Stopped))
        throw new Exception("Not all algs stopped properly");

      if (!algs.All(alg => ((DoubleValue)alg.Results["BestQuality"].Value).Value == ((DoubleValue)algs.First().Results["BestQuality"].Value).Value))
        throw new Exception("Not all algs have the same resutls");

      if (tb != null) {
        double totalExecutionTimeMilliseconds = algs.Select(x => x.ExecutionTime.TotalMilliseconds).Sum();
        double totalMilliseconds = tasks.Select(t => t.Result.TotalMilliseconds).Sum();
        tb.AppendRow(
          taskCount.ToString(),
          executionTimeUpdateIntervalMs.ToString(),
          TimeSpan.FromMilliseconds(totalExecutionTimeMilliseconds).ToString(),
          TimeSpan.FromMilliseconds(totalExecutionTimeMilliseconds / taskCount).ToString(),
          sw.Elapsed.ToString(),
          TimeSpan.FromMilliseconds(totalMilliseconds).ToString(),
          (totalMilliseconds / sw.ElapsedMilliseconds).ToString("0.00"),
          counter.ToString(),
          (totalExecutionTimeMilliseconds / counter).ToString("0.00"));
      }
      tasks = null;
      algs = null;
      GC.Collect();
      Console.WriteLine("Test finished.");
    }

    private static int counter = 0;
    static void Program_ExecutionTimeChanged(object sender, EventArgs e) {
      System.Threading.Interlocked.Increment(ref counter);
    }

    private static void TestWaitAny() {
      System.Random rand = new System.Random();
      var tasks = new List<Task<int>>();
      for (int i = 0; i < 10; i++) {
        tasks.Add(Task.Factory.StartNew<int>((x) => {
          int sleep = ((int)x - 10) * -1000;
          Console.WriteLine("sleeping: {0} ms", sleep);
          Thread.Sleep(0); // make context switch
          Thread.Sleep(sleep);
          return (int)x * (int)x;
        }, i));
      }

      // --> WaitAll processes tasks lazy but in order.
      Task.WaitAll();
      foreach (var task in tasks) {
        Console.WriteLine(task.Result);
      }

      // -> WaitAny processes any finished task first. but the finished task needs to be removed from list in order to process all tasks
      //for (int i = 0; i < 10; i++) {
      //  var tasksArray = tasks.ToArray();
      //  var task = tasksArray[Task.WaitAny(tasksArray)];
      //  Console.WriteLine(task.Result);
      //  tasks.Remove(task);
      //}

      Console.WriteLine("Finished TestWaitAny");
    }

    private static void TestAlgorithmPerformanceIssue() {
      Queue<TimeSpan> latestExecutionTimes = new Queue<TimeSpan>();
      int size = 10;
      var random = new Random.MersenneTwister(0);

      GeneticAlgorithm ga = new GeneticAlgorithm();
      ga.PopulationSize.Value = 3;
      ga.MaximumGenerations.Value = 1;
      ga.Engine = new SequentialEngine.SequentialEngine();
      ga.Problem = new SingleObjectiveTestFunctionProblem();

      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
      //metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
      ParameterConfigurationTree algorithmVc = SetupGAAlgorithm(typeof(GeneticAlgorithm), metaOptimizationProblem);
      algorithmVc.Randomize(random);
      Stopwatch sw = new Stopwatch();

      var algs = new Queue<IAlgorithm>(); // keep them in memory 
      // -> BINGO! -> .NET cannot hold more than 16 algorithms with their ThreadLocal<T> objects efficiently, 
      // so if they are kept in memory, runtime at the 17. execution drops significantly
      // because creating ThreadLocal<T> takes all the runtime.
      // when the algs are not stored in a list however this effect does not occur.


      for (int i = 0; i < 1000; i++) {
        GeneticAlgorithm clonedGa = (GeneticAlgorithm)ga.Clone();
        clonedGa.Name = "CLONED GA";
        //algorithmVc.Randomize(random);
        //algorithmVc.Parameterize(clonedGa);
        clonedGa.Prepare(true);
        sw.Start();
        algs.Enqueue(clonedGa);

        if (algs.Count > 24)
          algs.Dequeue();
        clonedGa.StartSync();
        sw.Stop();
        latestExecutionTimes.Enqueue(sw.Elapsed);
        Console.WriteLine("{0}: {1} ({2})", i, sw.Elapsed, latestExecutionTimes.Count > size ? TimeSpan.FromMilliseconds(latestExecutionTimes.Average(t => t.TotalMilliseconds)).ToString() : "-");
        if (latestExecutionTimes.Count > size) {
          latestExecutionTimes.Dequeue();
        }
        sw.Reset();
      }
    }

    private static void TestTableBuilder() {
      TableBuilder tb = new TableBuilder("column_1", "col2", "col3");
      tb.AppendRow("1", "humpi", "0.23124");
      tb.AppendRow("2", "sf", "0.23124");
      tb.AppendRow("5", "humpi dampti", "0.224");
      tb.AppendRow("10", "egon asdf", "0.4");
      tb.AppendRow("15", "MichaelizcMultiVfds", "0.23124564");
      Console.WriteLine(tb.ToString());
    }

    private static void TestToInfoString(IValueConfiguration algorithmVc) {
      var random = new MersenneTwister();
      Console.WriteLine(algorithmVc.ParameterInfoString);
      algorithmVc.Randomize(random);
      Console.WriteLine(algorithmVc.ParameterInfoString);
      algorithmVc.Randomize(random);
      Console.WriteLine(algorithmVc.ParameterInfoString);
      algorithmVc.Randomize(random);
    }

    private static void TestCombinations() {
      Console.WriteLine("IntRange 3-18:3");
      IntValueRange intRange = new IntValueRange(new IntValue(3), new IntValue(18), new IntValue(3));
      foreach (var val in intRange.GetCombinations()) {
        Console.WriteLine(val);
      }

      Console.WriteLine("DoubleRange 1.0-2.5:0.5");
      var dblRange = new DoubleValueRange(new DoubleValue(0.7), new DoubleValue(2.8), new DoubleValue(0.5));
      foreach (var val in dblRange.GetCombinations()) {
        Console.WriteLine(val);
      }

      Console.WriteLine("PercentRange 33%-66%:33%");
      var pctRange = new PercentValueRange(new PercentValue(0.32), new PercentValue(0.98), new PercentValue(0.33));
      foreach (var val in pctRange.GetCombinations()) {
        Console.WriteLine(val);
      }
    }

    private static void TestCombinations3() {
      Node root = new Node("root");
      root.ChildNodes.Add(new Node("root.n1"));
      root.ChildNodes.Add(new Node("root.n2"));
      Node n3 = new Node("root.n3");
      n3.ChildNodes.Add(new Node("root.n3.n1"));
      n3.ChildNodes.Add(new Node("root.n3.n2"));
      root.ChildNodes.Add(n3);

      Console.WriteLine(root.ToString());
      Console.WriteLine("--");
      int cnt = 0;
      var enumerator = new NodeEnumerator(root);
      enumerator.Reset();
      while (enumerator.MoveNext()) {
        Console.WriteLine(enumerator.Current.ToString());
        cnt++;
      }
      Console.WriteLine("count: " + cnt);
    }

    private static void TestEnumeratorCollectionEnumerator() {
      IEnumerable<int> list1 = new int[] { 1, 2, 3, 4, 5 };
      IEnumerable<int> list2 = new int[] { 10, 20, 30 };
      IEnumerable<int> list3 = new int[] { 300, 400, 500 };

      var enumerators = new List<IEnumerator>();

      EnumeratorCollectionEnumerator<int> enu = new EnumeratorCollectionEnumerator<int>();
      enu.AddEnumerator(list1.GetEnumerator());
      enu.AddEnumerator(list2.GetEnumerator());
      enu.AddEnumerator(list3.GetEnumerator());
      enu.Reset();
      while (enu.MoveNext()) {
        Console.WriteLine(enu.Current);
      }
    }

    private static void TestCombinations4() {
      GeneticAlgorithm ga = new GeneticAlgorithm();
      ga.Problem = new SingleObjectiveTestFunctionProblem();
      ga.Engine = new SequentialEngine.SequentialEngine();

      ParameterConfigurationTree vc = new ParameterConfigurationTree(ga, new SingleObjectiveTestFunctionProblem());

      ConfigurePopulationSize(vc, 20, 100, 20);
      //ConfigureMutationRate(vc, 0.10, 0.60, 0.10);
      ConfigureMutationOperator(vc);
      //ConfigureSelectionOperator(vc, true);

      int count = 0;
      IEnumerator enumerator = new ParameterCombinationsEnumerator(vc);
      enumerator.Reset();
      while (enumerator.MoveNext()) {
        var current = (IValueConfiguration)enumerator.Current;
        count++;
        Console.WriteLine(current.ParameterInfoString);
      }
      Console.WriteLine("You are about to create {0} algorithms.", count);

      Experiment experiment = vc.GenerateExperiment(ga);
      //foreach (var opt in experiment.Optimizers) {
      //  Console.WriteLine(opt.Name);
      //}

      experiment.Prepare();
      experiment.Start();

      while (experiment.ExecutionState != ExecutionState.Stopped) {
        Thread.Sleep(500);
      }
    }

    private static void TestOperators() {
      IRandom random = new MersenneTwister();

      var doubleRange = new DoubleValueRange(new DoubleValue(0), new DoubleValue(100), new DoubleValue(0.1));
      using (var sw = new StreamWriter("out-DoubleValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new DoubleValue(90);
          NormalDoubleValueManipulator.ApplyStatic(random, val, doubleRange);

          sw.WriteLine(val);
        }
      }

      var percentRange = new PercentValueRange(new PercentValue(0), new PercentValue(1), new PercentValue(0.001));
      using (var sw = new StreamWriter("out-PercentValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new PercentValue(0.5);
          NormalDoubleValueManipulator.ApplyStatic(random, val, percentRange.AsDoubleValueRange());
          sw.WriteLine(val);
        }
      }

      var intRange = new IntValueRange(new IntValue(0), new IntValue(100), new IntValue(1));
      using (var sw = new StreamWriter("out-IntValue.txt")) {
        for (int i = 0; i < 10000; i++) {
          var val = new IntValue(50);
          UniformIntValueManipulator.ApplyStatic(random, val, intRange);
          sw.WriteLine(val);
        }
      }

      Console.ReadLine();
    }

    private static void TestTypeDiscovery() {
      PluginLoader.pluginAssemblies.Any();

      var items = ApplicationManager.Manager.GetInstances(typeof(DoubleArray)).ToArray();

      foreach (var item in items) {
        Console.WriteLine(item.ToString());
      }
    }

    private static void TestMemoryLeak(GeneticAlgorithm metaLevelAlgorithm) {
      IValueConfiguration algorithmVc = ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).ParameterConfigurationTree;

      Console.WriteLine("Starting Memory Test...");
      Console.ReadLine();

      var clones = new List<object>();
      for (int i = 0; i < 1000; i++) {
        var clone = algorithmVc.Clone();
        clones.Add(clone);
      }

      Console.WriteLine("Finished. Now GC...");
      Console.ReadLine();

      GC.Collect();

      Console.WriteLine("Finished!");
      Console.ReadLine();
    }

    private static GeneticAlgorithm GetMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
      GeneticAlgorithm metaLevelAlgorithm = new GeneticAlgorithm();
      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;

      metaLevelAlgorithm.Problem = metaOptimizationProblem;
      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();

      metaLevelAlgorithm.Mutator = ((OptionalConstrainedValueParameter<IManipulator>)((IAlgorithm)metaLevelAlgorithm).Parameters["Mutator"]).ValidValues.Where(x => x.GetType() == typeof(ParameterConfigurationOnePositionsManipulator)).Single();
      //metaLevelAlgorithm.Mutator = ((OptionalConstrainedValueParameter<IManipulator>)((IAlgorithm)metaLevelAlgorithm).Parameters["Mutator"]).ValidValues.Where(x => x.GetType() == typeof(ParameterConfigurationAllPositionsManipulator)).Single();

      metaLevelAlgorithm.MutationProbability.Value = mutationProbability;
      //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(LinearRankSelector)).Single();
      //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(TournamentSelector)).Single();
      //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(GenderSpecificSelector)).Single();
      //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(BestSelector)).Single();
      metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(ProportionalSelector)).Single();

      return metaLevelAlgorithm;
    }

    private static GeneticAlgorithm GetParallelMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
      metaLevelAlgorithm.Engine = new ParallelEngine.ParallelEngine();
      return metaLevelAlgorithm;
    }

    private static GeneticAlgorithm GetHiveParallelMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
      GeneticAlgorithm metaLevelAlgorithm = GetParallelMetaGA(metaOptimizationProblem);
      metaLevelAlgorithm.Engine = new HiveEngine.HiveEngine();
      ServiceLocator.Instance.ClientFacadePool.UserName = "cneumuel";
      ServiceLocator.Instance.ClientFacadePool.Password = "cneumuel";
      ServiceLocator.Instance.StreamedClientFacadePool.UserName = "cneumuel";
      ServiceLocator.Instance.StreamedClientFacadePool.Password = "cneumuel";
      return metaLevelAlgorithm;
    }

    private static EvolutionStrategy GetMetaES(MetaOptimizationProblem metaOptimizationProblem) {
      EvolutionStrategy metaLevelAlgorithm = new EvolutionStrategy();
      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;

      metaLevelAlgorithm.Problem = metaOptimizationProblem;
      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();

      metaLevelAlgorithm.Mutator = ((OptionalConstrainedValueParameter<IManipulator>)((IAlgorithm)metaLevelAlgorithm).Parameters["Mutator"]).ValidValues.Last();

      return metaLevelAlgorithm;
    }

    private static ParameterConfigurationTree SetupGAAlgorithm(Type baseLevelAlgorithmType, MetaOptimizationProblem metaOptimizationProblem) {
      metaOptimizationProblem.AlgorithmType.Value = baseLevelAlgorithmType;
      //metaOptimizationProblem.Problems.Clear();

      //metaOptimizationProblem.ProblemType.Value = typeof(SingleObjectiveTestFunctionProblem);
      //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
      //  Evaluator = new GriewankEvaluator(),
      //  ProblemSize = new IntValue(2)
      //});
      //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
      //  Evaluator = new GriewankEvaluator(),
      //  ProblemSize = new IntValue(20)
      //});
      //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
      //  Evaluator = new GriewankEvaluator(),
      //  ProblemSize = new IntValue(500)
      //});

      metaOptimizationProblem.ProblemType.Value = typeof(SymbolicRegressionSingleObjectiveProblem);
      
      ParameterConfigurationTree algorithmVc = metaOptimizationProblem.ParameterConfigurationTree;
      ((IntValue)algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MaximumGenerations").ActualValue.Value).Value = baseAlgorithmMaxGenerations;

      //ConfigurePopulationSize(algorithmVc, 15, 20, 1);
      ConfigureMutationRate(algorithmVc, 0.0, 1.0, 0.01);
      ConfigureMutationOperator(algorithmVc);
      //ConfigureElites(algorithmVc, 0, 8, 1);
      //ConfigureSelectionOperator(algorithmVc, true);

      ConfigureSymbolicExpressionGrammar(algorithmVc);

      return algorithmVc;
    }

    private static void ConfigureSymbolicExpressionGrammar(ParameterConfigurationTree vc) {
      var pc = vc.ProblemConfiguration.ParameterConfigurations.Single(x => x.Name == "SymbolicExpressionTreeGrammar");
      pc.Optimize = true;

      SymbolicExpressionGrammarValueConfiguration symbolicExpressionGrammarVc = null;
      foreach (var valconf in pc.ValueConfigurations) {
        if (valconf.ActualValue.Value.ItemName != "TypeCoherentExpressionGrammar") {
          pc.ValueConfigurations.SetItemCheckedState(valconf, false);
        } else {
          symbolicExpressionGrammarVc = valconf as SymbolicExpressionGrammarValueConfiguration;
        }
      }
      symbolicExpressionGrammarVc.Optimize = true;

      var additionPc = symbolicExpressionGrammarVc.ParameterConfigurations.Single(x => x.Name == "Addition");
      additionPc.Optimize = true;
      var initialFrequencyVc = ((ParameterizedValueConfiguration)additionPc.ValueConfigurations.First()).ParameterConfigurations.Single(x => x.Name == "InitialFrequency");
      initialFrequencyVc.Optimize = true;
    }

    private static void TestConfiguration(ParameterConfigurationTree algorithmVc, Type baseLevelAlgorithmType, IProblem problem) {
      IRandom rand = new FastRandom(0);
      var baseLevelAlgorithm = (GeneticAlgorithm)MetaOptimizationUtil.CreateParameterizedAlgorithmInstance(algorithmVc, baseLevelAlgorithmType, problem);

      // set random values
      for (int i = 0; i < 10; i++) {
        var clonedVc = (ParameterConfigurationTree)algorithmVc.Clone();
        GeneticAlgorithm newAlg = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        clonedVc.Randomize(rand);
        clonedVc.Parameterize(newAlg);
        Console.WriteLine(string.Format("PopSize: original: {0}, randomized: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, randomized: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        //Console.WriteLine(string.Format("GrSi: original: {0}, randomized: {1}", "?", ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      algorithmVc.Randomize(rand);
      algorithmVc.Parameterize(baseLevelAlgorithm);
      // mutate
      for (int i = 0; i < 10; i++) {
        var clonedVc = (ParameterConfigurationTree)algorithmVc.Clone();
        GeneticAlgorithm newAlg = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        ParameterConfigurationManipulator.Apply(rand, clonedVc, new UniformIntValueManipulator(), new NormalDoubleValueManipulator());
        clonedVc.Parameterize(newAlg);

        Console.WriteLine(string.Format("PopSize: original: {0}, mutated: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
        Console.WriteLine(string.Format("MutRate: original: {0}, mutated: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
        Console.WriteLine(string.Format("MutOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
        Console.WriteLine(string.Format("SelOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
        //Console.WriteLine(string.Format("GrSi: original: {0}, mutated: {1}", ((TournamentSelector)baseLevelAlgorithm.Selector).GroupSizeParameter.Value, ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }

      Console.WriteLine("=======================");
      // cross
      for (int i = 0; i < 10; i++) {
        var clonedVc1 = (ParameterConfigurationTree)algorithmVc.Clone();
        var clonedVc2 = (ParameterConfigurationTree)algorithmVc.Clone();

        GeneticAlgorithm first = (GeneticAlgorithm)baseLevelAlgorithm.Clone();
        GeneticAlgorithm second = (GeneticAlgorithm)baseLevelAlgorithm.Clone();

        clonedVc1.Randomize(rand);
        clonedVc1.Parameterize(first);

        clonedVc2.Randomize(rand);
        clonedVc2.Parameterize(second);

        var popSizeBefore = first.PopulationSize.Value;
        var mutRateBefore = first.MutationProbability.Value;
        var mutOpBefore = first.Mutator;
        var selOpBefore = first.Selector;
        //var groupSizeBefore = ((TournamentSelector)first.Selector).GroupSizeParameter.Value.Value;

        //clonedVc1.Cross(clonedVc2, rand); todo

        ParameterConfigurationCrossover.Apply(rand, clonedVc1, clonedVc2, new DiscreteIntValueCrossover(), new AverageDoubleValueCrossover());
        clonedVc1.Parameterize(first);

        Console.WriteLine(string.Format("PopSize: first: {0}, second: {1}, crossed: {2}", popSizeBefore, second.PopulationSize, first.PopulationSize));
        Console.WriteLine(string.Format("MutRate: first: {0}, second: {1}, crossed: {2}", mutRateBefore, second.MutationProbability, first.MutationProbability));
        Console.WriteLine(string.Format("MutOp: first: {0}, second: {1}, crossed: {2}", mutOpBefore, second.Mutator, first.Mutator));
        Console.WriteLine(string.Format("SelOp: first: {0}, second: {1}, crossed: {2}", selOpBefore, second.Selector, first.Selector));
        //Console.WriteLine(string.Format("GrSi: first: {0}, second: {1}, crossed: {2}", groupSizeBefore, ((TournamentSelector)second.Selector).GroupSizeParameter.Value, ((TournamentSelector)first.Selector).GroupSizeParameter.Value));
        Console.WriteLine("---");
      }
      Console.WriteLine("=======================");
    }

    private static void ConfigureMutationOperator(ParameterConfigurationTree algorithmVc) {
      var mutationOperator = algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Where(x => x.Name == "Mutator").SingleOrDefault();
      mutationOperator.Optimize = true;

      // uncheck multiMutator to avoid Michalewicz issue
      //var multiMutator = mutationOperator.ValueConfigurations.Where(x => x.ActualValue.Value != null && x.ActualValue.Value.ItemName.StartsWith("Multi")).SingleOrDefault();
      //if (multiMutator != null) {
      //  mutationOperator.ValueConfigurations.SetItemCheckedState(multiMutator, false);
      //}

      // add another normal - don't do this with 'new', because ActualNames will not be set correctly. It should be copied from an existing one
      // mutationOperator.ValueConfigurations.Add(new ParameterizedValueConfiguration(new NormalAllPositionsManipulator(), typeof(NormalAllPositionsManipulator)), true);
    }

    private static void ConfigureSelectionOperator(ParameterConfigurationTree algorithmVc, bool configureTournamenSize) {
      var selectionOperatorPc = algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Where(x => x.Name == "Selector").SingleOrDefault();
      selectionOperatorPc.Optimize = true;

      foreach (var vc in selectionOperatorPc.ValueConfigurations) {
        if (vc.ActualValue.ValueDataType == typeof(TournamentSelector)) {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
          if (configureTournamenSize) {
            vc.Optimize = true;
            ConfigureTournamentGroupSize((ParameterizedValueConfiguration)vc);
          }
        } else if (vc.ActualValue.ValueDataType == typeof(RandomSelector)) {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
        } else {
          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
        }
      }
    }

    private static void ConfigureTournamentGroupSize(ParameterizedValueConfiguration tournamentVc) {
      var groupSizePc = tournamentVc.ParameterConfigurations.Where(x => x.ParameterName == "GroupSize").SingleOrDefault();
      groupSizePc.Optimize = true;
      var groupSizeVc = (RangeValueConfiguration)groupSizePc.ValueConfigurations.First();
      groupSizeVc.Optimize = true;
      groupSizeVc.RangeConstraint.LowerBound = new IntValue(0);
      groupSizeVc.RangeConstraint.UpperBound = new IntValue(10);
      groupSizeVc.RangeConstraint.StepSize = new IntValue(1);
    }

    private static void ConfigurePopulationSize(ParameterConfigurationTree algorithmVc, int lower, int upper, int stepsize) {
      var populationSizePc = algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Where(x => x.Name == "PopulationSize").SingleOrDefault();
      populationSizePc.Optimize = true;
      var populationSizeVc = (RangeValueConfiguration)populationSizePc.ValueConfigurations.First();
      populationSizeVc.Optimize = true;
      populationSizeVc.RangeConstraint.LowerBound = new IntValue(lower);
      populationSizeVc.RangeConstraint.UpperBound = new IntValue(upper);
      populationSizeVc.RangeConstraint.StepSize = new IntValue(stepsize);
    }

    private static void ConfigureMutationRate(ParameterConfigurationTree algorithmVc, double lower, double upper, double stepsize) {
      var mutationRatePc = algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Where(x => x.Name == "MutationProbability").SingleOrDefault();
      mutationRatePc.Optimize = true;
      var mutationRateVc = (RangeValueConfiguration)mutationRatePc.ValueConfigurations.First();
      mutationRateVc.Optimize = true;
      mutationRateVc.RangeConstraint.LowerBound = new PercentValue(lower);
      mutationRateVc.RangeConstraint.UpperBound = new PercentValue(upper);
      mutationRateVc.RangeConstraint.StepSize = new PercentValue(stepsize);
    }

    private static void ConfigureElites(ParameterConfigurationTree algorithmVc, int from, int to, int stepSize) {
      var elitesPc = algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Where(x => x.Name == "Elites").SingleOrDefault();
      elitesPc.Optimize = true;
      var elitesVc = (RangeValueConfiguration)elitesPc.ValueConfigurations.First();
      elitesVc.Optimize = true;
      elitesVc.RangeConstraint.LowerBound = new IntValue(from);
      elitesVc.RangeConstraint.UpperBound = new IntValue(to);
      elitesVc.RangeConstraint.StepSize = new IntValue(stepSize);
    }

    private static void TestOptimization(EngineAlgorithm metaLevelAlgorithm) {
      string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Results");
      if (!Directory.Exists(path))
        Directory.CreateDirectory(path);
      string id = DateTime.Now.ToString("yyyy.MM.dd - HH;mm;ss,ffff");
      string resultPath = Path.Combine(path, string.Format("{0} - Result.hl", id));
      string outputPath = Path.Combine(path, string.Format("{0} - Console.txt", id));

      using (var sw = new StreamWriter(outputPath)) {
        sw.AutoFlush = true;

        StringBuilder sb1 = new StringBuilder();
        sb1.AppendFormat("Meta.PopulationSize: {0}\n", metaAlgorithmPopulationSize);
        sb1.AppendFormat("Meta.MaxGenerations: {0}\n", metaAlgorithmMaxGenerations);
        sb1.AppendFormat("Meta.Repetitions   : {0}\n", metaProblemRepetitions);
        sb1.AppendFormat("Meta.MutProb       : {0}\n", ((GeneticAlgorithm)metaLevelAlgorithm).MutationProbability.Value);
        sb1.AppendFormat("Base.MaxGenerations: {0}\n", baseAlgorithmMaxGenerations);
        sb1.AppendLine("Problems:");
        foreach (var prob in ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).Problems) {
          sb1.Append(prob.Name);
          var sotf = prob as SingleObjectiveTestFunctionProblem;
          if (sotf != null) {
            sb1.AppendFormat(" {0}", sotf.ProblemSize.Value);
          }
          sb1.AppendLine();
        }
        sw.WriteLine(sb1.ToString());
        Console.WriteLine(sb1.ToString());
        metaLevelAlgorithm.Stopped += new EventHandler(metaLevelAlgorithm_Stopped);
        metaLevelAlgorithm.Paused += new EventHandler(metaLevelAlgorithm_Paused);
        metaLevelAlgorithm.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(metaLevelAlgorithm_ExceptionOccurred);

        metaLevelAlgorithm.Start();
        int i = 0;
        int currentGeneration = -1;
        do {
          Thread.Sleep(1000);
          if (metaLevelAlgorithm.Results.ContainsKey("Generations") && ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value != currentGeneration) {
            while (metaLevelAlgorithm.Results.Count < 6) Thread.Sleep(1000);
            StringBuilder sb = new StringBuilder();
            sb.AppendLine(DateTime.Now.ToLongTimeString());
            sb.AppendLine("=================================");

            sb.AppendLine(metaLevelAlgorithm.ExecutionState.ToString());
            ResultCollection rsClone = null;
            while (rsClone == null) {
              try {
                rsClone = (ResultCollection)metaLevelAlgorithm.Results.Clone();
              }
              catch { }
            }
            foreach (var result in rsClone) {
              sb.AppendLine(result.ToString());
              if (result.Name == "Population") {
                RunCollection rc = (RunCollection)result.Value;
                var orderedRuns = rc.OrderBy(x => x.Results["AverageQualityNormalized"]);

                TableBuilder tb = new TableBuilder("QNorm", "Qualities"/*, "PoSi"*/ ,"MutRa" /*,"Eli", "SelOp"*/, "MutOp"/*, "NrSelSubScopes"*/);
                foreach (IRun run in orderedRuns) {
                  //string selector;
                  //if (run.Parameters["Selector"] is TournamentSelector) {
                  //  selector = string.Format("{0} ({1})", run.Parameters["Selector"].ToString(), ((TournamentSelector)run.Parameters["Selector"]).GroupSizeParameter.Value.ToString());
                  //} else {
                  //  selector = string.Format("{0}", run.Parameters["Selector"].ToString());
                  //}

                  tb.AppendRow(
                    ((DoubleValue)run.Results["AverageQualityNormalized"]).Value.ToString("#0.0000")
                    ,((DoubleArray)run.Results["RunsAverageQualities"]).ToString()
                    //,((IntValue)run.Parameters["PopulationSize"]).Value.ToString()
                    ,((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.0000")
                    //,((IntValue)run.Parameters["Elites"]).Value.ToString()
                    //,Shorten(selector, 20)
                    ,Shorten(run.Parameters.ContainsKey("Mutator") ? run.Parameters["Mutator"].ToString() : "null", 40)
                    //,((ISelector)run.Parameters["Selector"]).NumberOfSelectedSubScopesParameter.Value.ToString()
                    );
                }
                sb.AppendLine(tb.ToString());
              }
            } // foreach
            //Console.Clear();
            Console.WriteLine(sb.ToString());
            sw.WriteLine(sb.ToString());
            currentGeneration = ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value;
          } // if
          //if (i % 30 == 0) GC.Collect();
          i++;
        } while (metaLevelAlgorithm.ExecutionState != ExecutionState.Stopped);
      }

      Console.WriteLine();
      Console.WriteLine("Storing...");

      ContentManager.Save((IStorableContent)metaLevelAlgorithm, resultPath, true);
      Console.WriteLine("Finished");
    }

    private static void metaLevelAlgorithm_ExceptionOccurred(object sender, EventArgs<Exception> e) {
      Console.WriteLine("metaLevelAlgorithm_ExceptionOccurred");
      Console.WriteLine(e.Value.ToString());
      if (e.Value.InnerException != null) {
        Console.WriteLine(e.Value.InnerException.ToString());
      }
    }

    private static void metaLevelAlgorithm_Paused(object sender, EventArgs e) {
      Console.WriteLine("metaLevelAlgorithm_Paused");
    }

    private static void metaLevelAlgorithm_Stopped(object sender, EventArgs e) {
      Console.WriteLine("metaLevelAlgorithm_Stopped");
    }

    private static void TestShorten() {
      int n = 8;
      Console.WriteLine(Shorten("1", n));
      Console.WriteLine(Shorten("12", n));
      Console.WriteLine(Shorten("123", n));
      Console.WriteLine(Shorten("1234", n));
      Console.WriteLine(Shorten("12345", n));
      Console.WriteLine(Shorten("123456", n));
      Console.WriteLine(Shorten("1234567", n));
      Console.WriteLine(Shorten("12345678", n));
      Console.WriteLine(Shorten("123456789", n));
      Console.WriteLine(Shorten("1234567890", n));
      Console.WriteLine(Shorten("12345678901", n));
    }

    private static string Shorten(string s, int n) {
      string placeholder = "..";
      if (s.Length <= n) return s;
      int len = n / 2 - placeholder.Length / 2;
      string start = s.Substring(0, len);
      string end = s.Substring(s.Length - len, len);
      return start + placeholder + end;
    }

    private static void TestIntSampling() {
      System.Random rand = new System.Random();
      int lower = 10;
      int upper = 20;
      int stepsize = 1;
      for (int i = 0; i < 100; i++) {
        int val;
        do {
          val = rand.Next(lower / stepsize, upper / stepsize + 1) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static void TestDoubleSampling() {
      System.Random rand = new System.Random();
      double lower = 2;
      double upper = 3;
      double stepsize = 0.6;
      for (int i = 0; i < 100; i++) {
        double val;
        do {
          val = Math.Round((rand.NextDouble() * (upper - lower) + lower) / stepsize, 0) * stepsize;
        } while (val < lower || val > upper);
        Console.WriteLine(val);
      }
    }

    private static IEnumerable<IItem> GetValidValues(IValueParameter valueParameter) {
      return ApplicationManager.Manager.GetInstances(valueParameter.DataType).Select(x => (IItem)x).OrderBy(x => x.ItemName);
    }
  }

  public class Node {
    public string Name { get; set; }
    public int ActualValue { get; set; }
    public int[] PossibleValues { get; set; }
    public List<Node> ChildNodes { get; set; }

    public Node(string name) {
      this.Name = name;
      PossibleValues = new int[] { 1, 2, 3 };
      ChildNodes = new List<Node>();
    }

    public void Init() {
      this.ActualValue = PossibleValues.First();
      foreach (var child in ChildNodes) {
        child.Init();
      }
    }

    public override string ToString() {
      StringBuilder sb = new StringBuilder();
      sb.Append(string.Format("{0}:{1}", this.Name, this.ActualValue));
      if (this.ChildNodes.Count() > 0) {
        sb.Append(" (");
        var lst = new List<string>();
        foreach (Node child in ChildNodes) {
          lst.Add(child.ToString());
        }
        sb.Append(string.Join(", ", lst.ToArray()));
        sb.Append(")");
      }

      return sb.ToString();
    }
  }

  public class NodeEnumerator : IEnumerator<Node> {
    private Node node;
    private List<IEnumerator> enumerators;

    public NodeEnumerator(Node node) {
      this.node = node;
      this.enumerators = new List<IEnumerator>();
    }

    public Node Current {
      get { return node; }
    }
    object IEnumerator.Current {
      get { return Current; }
    }

    public void Dispose() { }

    public bool MoveNext() {
      int i = 0;
      bool ok = false;
      while (!ok && i < enumerators.Count) {
        if (enumerators[i].MoveNext()) {
          ok = true;
        } else {
          i++;
        }
      }

      if (ok) {
        for (int k = i - 1; k >= 0; k--) {
          enumerators[k].Reset();
          enumerators[k].MoveNext();
        }
      } else {
        return false;
      }

      node.ActualValue = (int)enumerators[0].Current;
      return true;
    }

    public void Reset() {
      enumerators.Clear();
      enumerators.Add(node.PossibleValues.GetEnumerator());
      enumerators[0].Reset();

      foreach (var child in node.ChildNodes) {
        var enumerator = new NodeEnumerator(child);
        enumerator.Reset();
        enumerator.MoveNext();
        enumerators.Add(enumerator);
      }
    }
  }
}