Changeset 6090 for branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test

Timestamp:

05/01/11 22:02:28 (13 years ago)

Author:

cneumuel

Message:

#1215

• added weight parameters for quality, stddev and evaluated solutions
• lots of fixes

File:

• branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs (modified) (20 diffs)

branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs

@@ -15 +15 @@
 using HeuristicLab.Data;
 using HeuristicLab.Encodings.RealVectorEncoding;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 //using HeuristicLab.Hive.ExperimentManager;
 using HeuristicLab.Optimization;
@@ -21 +22 @@
 using HeuristicLab.PluginInfrastructure.Manager;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
-using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
 using HeuristicLab.Problems.MetaOptimization;
 using HeuristicLab.Problems.TestFunctions;
@@ -42 +42 @@
   [Application("TestApp")]
   public class TestApp : ApplicationBase {
-    //private static int metaAlgorithmPopulationSize = 50;
+    //private static int metaAlgorithmPopulationSize = 30;
     //private static int metaAlgorithmMaxGenerations = 30;
-    //private static int metaProblemRepetitions = 5;
-    //private static int baseAlgorithmMaxGenerations = 1000;
+    //private static int metaProblemRepetitions = 3;
+    //private static int baseAlgorithmMaxGenerations = 500;
+    //private static double mutationProbability = 0.10;
 
     private static int metaAlgorithmPopulationSize = 10;
     private static int metaAlgorithmMaxGenerations = 20;
+    private static double metaAlgorithmMutationProbability = 0.10;
     private static int metaProblemRepetitions = 2;
-    private static int baseAlgorithmMaxGenerations = 50;
-    private static double mutationProbability = 0.10;
-
+    private static int baseAlgorithmMaxGenerations = 10;
+    private static int baseAlgorithmPopulationSize = 10;
+    
     public override void Run() {
       ContentManager.Initialize(new PersistenceContentManager());
@@ -63 +65 @@
       //TestDoubleSampling(); return;
       //TestTypeDiscovery();
-      TestOperators(); return;
+      //TestOperators(); return;
       //TestCombinations();
       //TestCombinations2();
@@ -78 +80 @@
       //TestSymbolicDataAnalysisGrammar(); return;
 
-      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
+      //MetaOptimizationProblem metaOptimizationProblem = LoadOptimizationProblem("Meta Optimization Problem (Genetic Programming - Symbolic Regression 3.4 scaled).hl");
+      //var algorithmVc = metaOptimizationProblem.ParameterConfigurationTree;
+
+      var metaOptimizationProblem = new MetaOptimizationProblem();
+      var algorithmVc = SetupGAAlgorithm(typeof(GeneticAlgorithm), metaOptimizationProblem);
+
       metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
-      //GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
+      //GeneticAlgorithm metaLevelAlgorithm = GetSequentialMetaGA(metaOptimizationProblem);
       GeneticAlgorithm metaLevelAlgorithm = GetParallelMetaGA(metaOptimizationProblem);
       //GeneticAlgorithm metaLevelAlgorithm = GetHiveParallelMetaGA(metaOptimizationProblem);
-
       //EvolutionStrategy metaLevelAlgorithm = GetMetaES(metaOptimizationProblem);
-
-      var algorithmVc = SetupGAAlgorithm(typeof(GeneticAlgorithm), metaOptimizationProblem);
-
-      string info = algorithmVc.ParameterInfoString;
 
       //Console.WriteLine("Press enter to start");
@@ -103 +105 @@
     }
 
+    private MetaOptimizationProblem LoadOptimizationProblem(string filename) {
+      return (MetaOptimizationProblem)ContentManager.Load(filename);
+    }
+
     private void TestSymbolicDataAnalysisGrammar() {
       var random = new MersenneTwister();
@@ -108 +114 @@
       var grammar1 = new TypeCoherentExpressionGrammar();
       var grammar2 = new TypeCoherentExpressionGrammar();
-      
+
       Console.WriteLine("========== Grammar1: ==========");
       PrintGrammar(grammar1);
@@ -280 +286 @@
       MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
       metaOptimizationProblem.Repetitions = new IntValue(metaProblemRepetitions);
-      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
+      GeneticAlgorithm metaLevelAlgorithm = GetSequentialMetaGA(metaOptimizationProblem);
       ParameterConfigurationTree algorithmVc = SetupGAAlgorithm(typeof(GeneticAlgorithm), metaOptimizationProblem);
       Stopwatch sw = new Stopwatch();
@@ -459 +465 @@
 
 
-      for (int i = 0; i < 1000; i++) {
+      for (int i = 0; i < 10000; i++) {
         GeneticAlgorithm clonedGa = (GeneticAlgorithm)ga.Clone();
         clonedGa.Name = "CLONED GA";
@@ -479 +485 @@
         }
         sw.Reset();
+        //Console.ReadLine();
       }
     }
@@ -670 +677 @@
     }
 
-    private static GeneticAlgorithm GetMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
+    private static GeneticAlgorithm GetSequentialMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
       GeneticAlgorithm metaLevelAlgorithm = new GeneticAlgorithm();
       metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
@@ -681 +688 @@
       //metaLevelAlgorithm.Mutator = ((OptionalConstrainedValueParameter<IManipulator>)((IAlgorithm)metaLevelAlgorithm).Parameters["Mutator"]).ValidValues.Where(x => x.GetType() == typeof(ParameterConfigurationAllPositionsManipulator)).Single();
 
-      metaLevelAlgorithm.MutationProbability.Value = mutationProbability;
+      metaLevelAlgorithm.MutationProbability.Value = metaAlgorithmMutationProbability;
       //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(LinearRankSelector)).Single();
       //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(TournamentSelector)).Single();
       //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(GenderSpecificSelector)).Single();
       //metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(BestSelector)).Single();
-      metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(ProportionalSelector)).Single();
+      metaLevelAlgorithm.Selector = ((OptionalConstrainedValueParameter<ISelector>)((IAlgorithm)metaLevelAlgorithm).Parameters["Selector"]).ValidValues.Where(x => x.GetType() == typeof(TournamentSelector)).Single();
+
+      metaLevelAlgorithm.SetSeedRandomly.Value = false;
+      //metaLevelAlgorithm.Seed.Value = new MersenneTwister().Next(0, 1000000);
+      metaLevelAlgorithm.Seed.Value = 527875;
 
       return metaLevelAlgorithm;
@@ -692 +703 @@
 
     private static GeneticAlgorithm GetParallelMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
-      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
+      GeneticAlgorithm metaLevelAlgorithm = GetSequentialMetaGA(metaOptimizationProblem);
       metaLevelAlgorithm.Engine = new ParallelEngine.ParallelEngine();
       return metaLevelAlgorithm;
@@ -722 +733 @@
     private static ParameterConfigurationTree SetupGAAlgorithm(Type baseLevelAlgorithmType, MetaOptimizationProblem metaOptimizationProblem) {
       metaOptimizationProblem.AlgorithmType.Value = baseLevelAlgorithmType;
+      
+      //metaOptimizationProblem.ProblemType.Value = typeof(SingleObjectiveTestFunctionProblem);
       //metaOptimizationProblem.Problems.Clear();
-
-      //metaOptimizationProblem.ProblemType.Value = typeof(SingleObjectiveTestFunctionProblem);
       //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
       //  Evaluator = new GriewankEvaluator(),
@@ -731 +742 @@
       //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
       //  Evaluator = new GriewankEvaluator(),
-      //  ProblemSize = new IntValue(20)
+      //  ProblemSize = new IntValue(50)
       //});
       //metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
@@ -738 +749 @@
       //});
 
-      metaOptimizationProblem.ProblemType.Value = typeof(SymbolicRegressionSingleObjectiveProblem);
-      
+      //metaOptimizationProblem.ProblemType.Value = typeof(SymbolicRegressionSingleObjectiveProblem);
+      //metaOptimizationProblem.Maximization.Value = true;
+
+      // tower problem
+      metaOptimizationProblem.ImportAlgorithm((IAlgorithm)ContentManager.Load("Genetic Programming - Symbolic Regression 3.4_scaled.hl"));
+      metaOptimizationProblem.Maximization.Value = true;
+
       ParameterConfigurationTree algorithmVc = metaOptimizationProblem.ParameterConfigurationTree;
       ((IntValue)algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "MaximumGenerations").ActualValue.Value).Value = baseAlgorithmMaxGenerations;
-
-      //ConfigurePopulationSize(algorithmVc, 15, 20, 1);
+      ((IntValue)algorithmVc.AlgorithmConfiguration.ParameterConfigurations.Single(x => x.Name == "PopulationSize").ActualValue.Value).Value = baseAlgorithmPopulationSize;
+      
+      //ConfigurePopulationSize(algorithmVc, 10, 100, 1);
       //ConfigureMutationRate(algorithmVc, 0.0, 1.0, 0.01);
       //ConfigureMutationOperator(algorithmVc);
-      //ConfigureElites(algorithmVc, 0, 8, 1);
+      //ConfigureElites(algorithmVc, 0, 10, 1);
       //ConfigureSelectionOperator(algorithmVc, true);
 
@@ -772 +789 @@
     private static void ConfigureSymbolicExpressionGrammarVc(SymbolicExpressionGrammarValueConfiguration symbolicExpressionGrammarVc) {
       symbolicExpressionGrammarVc.Optimize = true;
-      foreach (var pc in symbolicExpressionGrammarVc.ParameterConfigurations) {
-        if (pc.Name != "Constant"
-          && pc.Name != "Variable" 
-          && pc.Name != "ProgramRootSymbol"
-          && pc.Name != "StartSymbol") {
-          pc.Optimize = true;
-        }
-      }
-      //var additionPc = symbolicExpressionGrammarVc.ParameterConfigurations.Single(x => x.Name == "Addition");
-      //additionPc.Optimize = true;
+
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Addition", 1.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Subtraction", 1.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Multiplication", 1.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Division", 1.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Average", 1.0);
+
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "IfThenElse", 0.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "GreaterThan", 0.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "LessThan", 0.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "And", 0.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Or", 0.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Not", 0.0);
+
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Sine", 0.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Cosine", 0.0);
+      SetInitialFrequencyValue(symbolicExpressionGrammarVc, "Tangent", 0.0);
+
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Logarithm", "InitialFrequency", 0.0, 5.0, 0.01);
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Power", "InitialFrequency", 0.0, 5.0, 0.01);
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Root", "InitialFrequency", 0.0, 5.0, 0.01);
+
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Variable", "WeightSigma", 0.01, 10.0, 0.01);
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Variable", "WeightManipulatorSigma", 0.01, 10.0, 0.01);
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Variable", "MultiplicativeWeightManipulatorSigma", 0.01, 10.0, 0.01);
+
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Constant", "ManipulatorSigma", 0.01, 10.0, 0.01);
+      OptimizeInitialFrequency(symbolicExpressionGrammarVc, "Constant", "MultiplicativeManipulatorSigma", 0.01, 10.0, 0.01);
+    }
+
+    private static void SetInitialFrequencyValue(SymbolicExpressionGrammarValueConfiguration symbolicExpressionGrammarVc, string symbolName, double value) {
+      ((Symbol)symbolicExpressionGrammarVc.ParameterConfigurations.Single(x => x.Name == symbolName).ActualValue.Value).InitialFrequency = value;
+    }
+
+    private static void OptimizeInitialFrequency(SymbolicExpressionGrammarValueConfiguration symbolicExpressionGrammarVc, string symbolName, string parameterName, double lower, double upper, double step) {
+      var pc = symbolicExpressionGrammarVc.ParameterConfigurations.Single(x => x.Name == symbolName);
+      pc.Optimize = true;
+      var vc = (SymbolValueConfiguration)pc.ValueConfigurations.Single();
+      var parameterPc = vc.ParameterConfigurations.Single(x => x.Name == parameterName);
+      parameterPc.Optimize = true;
+      parameterPc.ValueConfigurations.Clear();
+      var rvc = new RangeValueConfiguration(new DoubleValue(5.0), typeof(DoubleValue));
+      rvc.Optimize = true;
+      ((DoubleValueRange)rvc.RangeConstraint).LowerBound.Value = lower;
+      ((DoubleValueRange)rvc.RangeConstraint).UpperBound.Value = upper;
+      ((DoubleValueRange)rvc.RangeConstraint).StepSize.Value = step;
+      parameterPc.ValueConfigurations.Add(rvc);
     }
 
@@ -937 +991 @@
       string outputPath = Path.Combine(path, string.Format("{0} - Console.txt", id));
 
-      ContentManager.Save((IStorableContent)metaLevelAlgorithm, resultPath+"-before.hl", true);
-
+      ContentManager.Save((IStorableContent)metaLevelAlgorithm, resultPath + "-before.hl", true);
+      
       using (var sw = new StreamWriter(outputPath)) {
         sw.AutoFlush = true;
@@ -947 +1001 @@
         sb1.AppendFormat("Meta.Repetitions   : {0}\n", metaProblemRepetitions);
         sb1.AppendFormat("Meta.MutProb       : {0}\n", ((GeneticAlgorithm)metaLevelAlgorithm).MutationProbability.Value);
+        sb1.AppendFormat("Meta.Seed          : {0}\n", ((GeneticAlgorithm)metaLevelAlgorithm).Seed.Value);
         sb1.AppendFormat("Base.MaxGenerations: {0}\n", baseAlgorithmMaxGenerations);
+
         sb1.AppendLine("Problems:");
         foreach (var prob in ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).Problems) {
@@ -988 +1044 @@
                 var orderedRuns = rc.OrderBy(x => x.Results["AverageQualityNormalized"]);
 
-                TableBuilder tb = new TableBuilder("QNorm", "Qualities"/*, "PoSi"*/ /*,"MutRa"*/ /*,"Eli", "SelOp",*/ /*"MutOp"*//*, "NrSelSubScopes"*/);
+                //TableBuilder tb = new TableBuilder("QNorm", "Qualities"/*, "PoSi"*/ /*,"MutRa"*/ /*,"Eli", "SelOp",*/ /*"MutOp"*//*, "NrSelSubScopes"*/);
+                //foreach (IRun run in orderedRuns) {
+                //  //string selector;
+                //  //if (run.Parameters["Selector"] is TournamentSelector) {
+                //  //  selector = string.Format("{0} ({1})", run.Parameters["Selector"].ToString(), ((TournamentSelector)run.Parameters["Selector"]).GroupSizeParameter.Value.ToString());
+                //  //} else {
+                //  //  selector = string.Format("{0}", run.Parameters["Selector"].ToString());
+                //  //}
+
+                //  tb.AppendRow(
+                //    ((DoubleValue)run.Results["AverageQualityNormalized"]).Value.ToString("#0.0000")
+                //    ,((DoubleArray)run.Results["RunsAverageQualities"]).ToString()
+                //    //,((IntValue)run.Parameters["PopulationSize"]).Value.ToString()
+                //    //,((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.0000")
+                //    //,((IntValue)run.Parameters["Elites"]).Value.ToString()
+                //    //,Shorten(selector, 20)
+                //    //,Shorten(run.Parameters.ContainsKey("Mutator") ? run.Parameters["Mutator"].ToString() : "null", 40)
+                //    //,((ISelector)run.Parameters["Selector"]).NumberOfSelectedSubScopesParameter.Value.ToString()
+                //    );
+                //}
+                //sb.AppendLine(tb.ToString());
+
+                var tb = new TableBuilder("QNorm", "Qualities", "StdDevs", "Evaluations", "Parameters");
                 foreach (IRun run in orderedRuns) {
-                  //string selector;
-                  //if (run.Parameters["Selector"] is TournamentSelector) {
-                  //  selector = string.Format("{0} ({1})", run.Parameters["Selector"].ToString(), ((TournamentSelector)run.Parameters["Selector"]).GroupSizeParameter.Value.ToString());
-                  //} else {
-                  //  selector = string.Format("{0}", run.Parameters["Selector"].ToString());
-                  //}
-
                   tb.AppendRow(
                     ((DoubleValue)run.Results["AverageQualityNormalized"]).Value.ToString("#0.0000")
-                    ,((DoubleArray)run.Results["RunsAverageQualities"]).ToString()
-                    //,((IntValue)run.Parameters["PopulationSize"]).Value.ToString()
-                    //,((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.0000")
-                    //,((IntValue)run.Parameters["Elites"]).Value.ToString()
-                    //,Shorten(selector, 20)
-                    //,Shorten(run.Parameters.ContainsKey("Mutator") ? run.Parameters["Mutator"].ToString() : "null", 40)
-                    //,((ISelector)run.Parameters["Selector"]).NumberOfSelectedSubScopesParameter.Value.ToString()
+                    , ((DoubleArray)run.Results["RunsAverageQualities"]).ToString()
+                    , ((DoubleArray)run.Results["RunsQualityStandardDeviations"]).ToString()
+                    , ((DoubleArray)run.Results["RunsAverageEvaluatedSolutions"]).ToString()
+                    , run.Name
                     );
                 }