Changeset 5207 for branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs

Timestamp:

01/04/11 02:18:27 (13 years ago)

Author:

cneumuel

Message:

#1215

• lots of memory-consumption improvements
• validValues -> validTypes (this saves memory!)
• changed manipulators; modifications are less significant, out-of-bound-values are resampled instead of set to lower or upper bound
• changed the way a base-level algorithm gets executed -> introduced AlgorithmExecutor

File:

• branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs (modified) (15 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs

@@ -18 +18 @@
 using HeuristicLab.Selection;
 using HeuristicLab.Parameters;
+using HeuristicLab.Operators;
+using System.Diagnostics;
+using HeuristicLab.Encodings.RealVectorEncoding;
 
 namespace HeuristicLab.MetaOptimization.Test {
   class Program {
-    private static int metaAlgorithmPopulationSize = 10;
-    private static int metaAlgorithmMaxGenerations = 10;
-    private static int metaProblemRepetitions = 1;
-
-    private static int baseAlgorithmMaxGenerations = 10;
+    private static int metaAlgorithmPopulationSize = 50;
+    private static int metaAlgorithmMaxGenerations = 30;
+    private static int metaProblemRepetitions = 5;
+    private static int baseAlgorithmMaxGenerations = 1000;
+
+    //private static int metaAlgorithmPopulationSize = 10;
+    //private static int metaAlgorithmMaxGenerations = 20;
+    //private static int metaProblemRepetitions = 3;
+    //private static int baseAlgorithmMaxGenerations = 10;
 
     static void Main(string[] args) {
+      //TestTableBuilder();
       //TestShorten();
 
       //TestIntSampling();
-      //TestDoubleSampling();
+      //TestDoubleSampling(); return;
       //TestTypeDiscovery();
       //TestOperators();
@@ -39 +47 @@
       //TestEnumeratorCollectionEnumerator();
       //TestCombinations4();
+      //TestAlgorithmPerformanceIssue();
 
       GeneticAlgorithm baseLevelAlgorithm = new GeneticAlgorithm();
+
+      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
+      metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
+      //GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
+      GeneticAlgorithm metaLevelAlgorithm = GetParallelMetaGA(metaOptimizationProblem);
+      //EvolutionStrategy metaLevelAlgorithm = GetMetaES(metaOptimizationProblem);
+
+      IValueConfiguration algorithmVc = SetupGAAlgorithm(baseLevelAlgorithm, metaOptimizationProblem);
+
+      //TestToString(algorithmVc);
+
+
+      //Console.WriteLine("Press enter to start");
+      //Console.ReadLine();
+      //TestConfiguration(algorithmVc, baseLevelAlgorithm);
+
+      //Console.WriteLine("Press enter to start");
+      //Console.ReadLine();
+      TestOptimization(metaLevelAlgorithm);
+
+      //TestMemoryLeak(metaLevelAlgorithm);
+
+      Console.ReadLine();
+    }
+
+    private static void TestAlgorithmPerformanceIssue() {
+      ContentManager.Initialize(new PersistenceContentManager());
+      Queue<TimeSpan> latestExecutionTimes = new Queue<TimeSpan>();
+      int size = 10;
+
+      GeneticAlgorithm ga = new GeneticAlgorithm();
+      ga.PopulationSize.Value = 3;
+      ga.MaximumGenerations.Value = 1;
+      ga.Engine = new SequentialEngine.SequentialEngine();
 
       MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
       metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
       GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
-      //EvolutionStrategy metaLevelAlgorithm = GetMetaES(metaOptimizationProblem);
-
-      IValueConfiguration algorithmVc = SetupGAAlgorithm(baseLevelAlgorithm, metaOptimizationProblem);
-
-      //TestToString(algorithmVc);
-
-
-      //Console.WriteLine("Press enter to start");
-      //Console.ReadLine();
-      //TestConfiguration(algorithmVc, baseLevelAlgorithm);
-
-      //Console.WriteLine("Press enter to start");
-      //Console.ReadLine();
-      TestOptimization(metaLevelAlgorithm);
-
-      //TestMemoryLeak(metaLevelAlgorithm);
-
-      Console.ReadLine();
-    }
-
-    private static void TestToString(IValueConfiguration algorithmVc) {
+      ParameterConfigurationTree algorithmVc = SetupGAAlgorithm(ga, metaOptimizationProblem);
+      Stopwatch sw = new Stopwatch();
+
+      for (int i = 0; i < 1000; i++) {
+        sw.Start();
+        GeneticAlgorithm clonedGa = (GeneticAlgorithm)ga.Clone();
+        clonedGa.Name = "CLONED GA";
+        algorithmVc.Parameterize(clonedGa);
+        clonedGa.Prepare(true);
+        var executor = new AlgorithmExecutor(clonedGa);
+        executor.StartSync();
+        sw.Stop();
+        latestExecutionTimes.Enqueue(sw.Elapsed);
+        Console.WriteLine("{0}: {1} ({2})", i, sw.Elapsed, latestExecutionTimes.Count > size ? TimeSpan.FromMilliseconds(latestExecutionTimes.Average(t => t.TotalMilliseconds)).ToString() : "-");
+        if (latestExecutionTimes.Count > size) {
+          latestExecutionTimes.Dequeue();
+        }
+        sw.Reset();
+      }
+    }
+
+    private static void TestTableBuilder() {
+      TableBuilder tb = new TableBuilder("column_1", "col2", "col3");
+      tb.AppendRow("1", "humpi", "0.23124");
+      tb.AppendRow("2", "sf", "0.23124");
+      tb.AppendRow("5", "humpi dampti", "0.224");
+      tb.AppendRow("10", "egon asdf", "0.4");
+      tb.AppendRow("15", "MichaelizcMultiVfds", "0.23124564");
+      Console.WriteLine(tb.ToString());
+    }
+
+    private static void TestToInfoString(IValueConfiguration algorithmVc) {
       var random = new MersenneTwister();
       Console.WriteLine(algorithmVc.ParameterInfoString);
@@ -154 +208 @@
       }
       Console.WriteLine("You are about to create {0} algorithms.", count);
-      
+
       Experiment experiment = vc.GenerateExperiment(ga);
       //foreach (var opt in experiment.Optimizers) {
@@ -170 +224 @@
     private static void TestOperators() {
       IRandom random = new MersenneTwister();
-      ParameterConfigurationManipulator manip = new ParameterConfigurationManipulator();
-
-      manip.IntValueManipulatorParameter.ActualValue = new UniformIntValueManipulator();
-      manip.DoubleValueManipulatorParameter.ActualValue = new NormalDoubleValueManipulator();
 
       var doubleRange = new DoubleValueRange(new DoubleValue(0), new DoubleValue(100), new DoubleValue(0.1));
       using (var sw = new StreamWriter("out-DoubleValue.txt")) {
         for (int i = 0; i < 10000; i++) {
-          var val = new DoubleValue(50);
+          var val = new DoubleValue(90);
           NormalDoubleValueManipulator.ApplyStatic(random, val, doubleRange);
 
@@ -244 +294 @@
       metaLevelAlgorithm.Problem = metaOptimizationProblem;
       metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();
-      
+
       metaLevelAlgorithm.Mutator = ((OptionalConstrainedValueParameter<IManipulator>)((IAlgorithm)metaLevelAlgorithm).Parameters["Mutator"]).ValidValues.Last();
-      
+
       metaLevelAlgorithm.MutationProbability.Value = 0.15;
+
+      return metaLevelAlgorithm;
+    }
+
+    private static GeneticAlgorithm GetParallelMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
+      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
+
+      UniformSubScopesProcessor uniformSubScopesProcessor =
+        (HeuristicLab.Operators.UniformSubScopesProcessor)
+        ((HeuristicLab.Operators.AlgorithmOperator)metaLevelAlgorithm.OperatorGraph.Operators.ElementAt(2)).OperatorGraph.Operators.ElementAt(8);
+      uniformSubScopesProcessor.Parallel.Value = true;
+
+      metaLevelAlgorithm.Engine = new ParallelEngine.ParallelEngine();
 
       return metaLevelAlgorithm;
@@ -265 +328 @@
     }
 
-    private static IValueConfiguration SetupGAAlgorithm(GeneticAlgorithm baseLevelAlgorithm, MetaOptimizationProblem metaOptimizationProblem) {
+    private static ParameterConfigurationTree SetupGAAlgorithm(GeneticAlgorithm baseLevelAlgorithm, MetaOptimizationProblem metaOptimizationProblem) {
       baseLevelAlgorithm.Problem = new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem();
       baseLevelAlgorithm.MaximumGenerations.Value = baseAlgorithmMaxGenerations;
 
       metaOptimizationProblem.Algorithm = baseLevelAlgorithm;
-      IValueConfiguration algorithmVc = metaOptimizationProblem.ParameterConfigurationTree;
+      ParameterConfigurationTree algorithmVc = metaOptimizationProblem.ParameterConfigurationTree;
 
       metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
@@ -276 +339 @@
         ProblemSize = new IntValue(500)
       });
-      metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
-        Evaluator = new GriewankEvaluator(),
-        ProblemSize = new IntValue(1000)
-      });
-
-      ConfigurePopulationSize(algorithmVc, 0, 10, 1);
-      //ConfigureMutationRate(algorithmVc, 0.0, 1.0, 0.01);
-      //ConfigureMutationOperator(algorithmVc);
+      //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
+      //  Evaluator = new GriewankEvaluator(),
+      //  ProblemSize = new IntValue(1000)
+      //});
+
+      ConfigurePopulationSize(algorithmVc, 0, 100, 1);
+      ConfigureMutationRate(algorithmVc, 0.0, 1.0, 0.01);
+      ConfigureMutationOperator(algorithmVc);
       ConfigureElites(algorithmVc, 0, 10, 1);
-      //ConfigureSelectionOperator(algorithmVc, true);
+      ConfigureSelectionOperator(algorithmVc, true);
       return algorithmVc;
     }
@@ -393 +456 @@
 
       groupSizePc.ValueConfigurations.First().Optimize = true;
-      groupSizePc.ValueConfigurations.First().RangeConstraint.LowerBound = new IntValue(2);
+      groupSizePc.ValueConfigurations.First().RangeConstraint.LowerBound = new IntValue(0);
       groupSizePc.ValueConfigurations.First().RangeConstraint.UpperBound = new IntValue(10);
       groupSizePc.ValueConfigurations.First().RangeConstraint.StepSize = new IntValue(1);
@@ -441 +504 @@
 
         StringBuilder sb1 = new StringBuilder();
-        sb1.AppendLine(string.Format("Meta.PopulationSize: {0}", metaAlgorithmPopulationSize));
-        sb1.AppendLine(string.Format("Meta.MaxGenerations: {0}", metaAlgorithmMaxGenerations));
-        sb1.AppendLine(string.Format("Meta.Repetitions   : {0}", metaProblemRepetitions));
-        sb1.AppendLine(string.Format("Base.MaxGenerations: {0}", baseAlgorithmMaxGenerations));
+        sb1.AppendFormat("Meta.PopulationSize: {0}\n", metaAlgorithmPopulationSize);
+        sb1.AppendFormat("Meta.MaxGenerations: {0}\n", metaAlgorithmMaxGenerations);
+        sb1.AppendFormat("Meta.Repetitions   : {0}\n", metaProblemRepetitions);
+        sb1.AppendFormat("Meta.MutProb       : {0}\n", ((GeneticAlgorithm)metaLevelAlgorithm).MutationProbability.Value);
+        sb1.AppendFormat("Base.MaxGenerations: {0}\n", baseAlgorithmMaxGenerations);
+        sb1.AppendLine("Problems:");
+        foreach (var prob in ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).Problems) {
+          sb1.Append(prob.Name);
+          var sotf = prob as SingleObjectiveTestFunctionProblem;
+          if (sotf != null) {
+            sb1.AppendFormat(" {0}", sotf.ProblemSize.Value);
+          }
+          sb1.AppendLine();
+        }
         sw.WriteLine(sb1.ToString());
         Console.WriteLine(sb1.ToString());
@@ -463 +536 @@
 
             sb.AppendLine(metaLevelAlgorithm.ExecutionState.ToString());
-            foreach (var result in metaLevelAlgorithm.Results) {
+            var rsClone = (ResultCollection)metaLevelAlgorithm.Results.Clone();
+            foreach (var result in rsClone) {
               sb.AppendLine(result.ToString());
               if (result.Name == "Population") {
@@ -469 +543 @@
                 var orderedRuns = rc.OrderBy(x => x.Results["RunsAverageQuality"]);
 
-                sb.AppendLine("Qual.  PoSi MutRa Eli SelOp MutOp");
+                TableBuilder tb = new TableBuilder("AvgQual", "AvgET", "PoSi", "MutRa", "Eli", "SelOp", "MutOp", "NrSelSubScopes");
                 foreach (IRun run in orderedRuns) {
                   string selector;
@@ -478 +552 @@
                   }
 
-                  sb.AppendLine(string.Format("{0} {1} {2} {3} {4} {5}",
-                    ((DoubleValue)run.Results["RunsAverageQuality"]).Value.ToString("#0.00").PadLeft(7, ' '),
-                    ((IntValue)run.Parameters["PopulationSize"]).Value.ToString().PadLeft(3, ' ').PadRight(3, ' '),
-                    ((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.00").PadLeft(5, ' '),
-                    ((IntValue)run.Parameters["Elites"]).Value.ToString().PadLeft(3, ' '),
-                    Shorten(selector, 20).PadRight(20, ' '),
-                    run.Parameters.ContainsKey("Mutator") ? run.Parameters["Mutator"].ToString() : "null"));
+                  tb.AppendRow(
+                    ((DoubleValue)run.Results["RunsAverageQuality"]).Value.ToString("#0.00"),
+                    run.Results["RunsAverageExecutionTime"].ToString(),
+                    ((IntValue)run.Parameters["PopulationSize"]).Value.ToString(),
+                    ((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.00"),
+                    ((IntValue)run.Parameters["Elites"]).Value.ToString(),
+                    Shorten(selector, 20),
+                    Shorten(run.Parameters.ContainsKey("Mutator") ? run.Parameters["Mutator"].ToString() : "null", 40),
+                    ((ISelector)run.Parameters["Selector"]).NumberOfSelectedSubScopesParameter.Value.ToString());
                 }
+                sb.AppendLine(tb.ToString());
               }
             } // foreach
-            Console.Clear();
+            //Console.Clear();
             Console.WriteLine(sb.ToString());
             sw.WriteLine(sb.ToString());
             currentGeneration = ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value;
           } // if
-          if (i % 30 == 0) GC.Collect();
+          //if (i % 30 == 0) GC.Collect();
           i++;
         } while (metaLevelAlgorithm.ExecutionState != ExecutionState.Stopped);
@@ -545 +622 @@
       int lower = 10;
       int upper = 20;
-      int stepsize = 7;
+      int stepsize = 1;
       for (int i = 0; i < 100; i++) {
         int val;
@@ -631 +708 @@
       int i = 0;
       bool ok = false;
-      while(!ok && i < enumerators.Count) {
-        if(enumerators[i].MoveNext()) {
+      while (!ok && i < enumerators.Count) {
+        if (enumerators[i].MoveNext()) {
           ok = true;
         } else {
@@ -640 +717 @@
 
       if (ok) {
-        for (int k = i-1; k >= 0; k--) {
+        for (int k = i - 1; k >= 0; k--) {
           enumerators[k].Reset();
           enumerators[k].MoveNext();