Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.MetaOptimization/HeuristicLab.MetaOptimization.Test/Program.cs @ 5087

Last change on this file since 5087 was 5087, checked in by cneumuel, 13 years ago

#1215

  • enabled multiple problems
  • enabled n repetitions
  • improved results output
  • reduced memory footprint significantly
  • removed viewhost icons for less screen space waste
File size: 18.7 KB
Line 
1using System.Collections.Generic;
2using System.Diagnostics;
3using System.Linq;
4using HeuristicLab.Algorithms.GeneticAlgorithm;
5using HeuristicLab.Core;
6using HeuristicLab.PluginInfrastructure;
7using HeuristicLab.Parameters;
8using HeuristicLab.Problems.MetaOptimization;
9using HeuristicLab.Data;
10using System;
11using System.Threading;
12using HeuristicLab.Random;
13using HeuristicLab.Optimization;
14using HeuristicLab.Common;
15using System.IO;
16using HeuristicLab.Problems.TestFunctions;
17using System.Text;
18using HeuristicLab.Selection;
19using HeuristicLab.Algorithms.EvolutionStrategy;
20
21namespace HeuristicLab.MetaOptimization.Test {
22  class Program {
23    private static int metaAlgorithmPopulationSize = 50;
24    private static int metaAlgorithmMaxGenerations = 30;
25    private static int metaProblemRepetitions = 3;
26
27    private static int baseAlgorithmMaxGenerations = 250;
28
29    static void Main(string[] args) {
30      //TestShorten();
31
32      //TestIntSampling();
33      //TestDoubleSampling();
34
35      GeneticAlgorithm baseLevelAlgorithm = new GeneticAlgorithm();
36
37      MetaOptimizationProblem metaOptimizationProblem = new MetaOptimizationProblem();
38      ((MetaOptimizationEvaluator)metaOptimizationProblem.Evaluator).Repetitions.Value = metaProblemRepetitions;
39      GeneticAlgorithm metaLevelAlgorithm = GetMetaGA(metaOptimizationProblem);
40      //EvolutionStrategy metaLevelAlgorithm = GetMetaES(metaOptimizationProblem);
41
42      IValueConfiguration algorithmVc = SetupGAAlgorithm(baseLevelAlgorithm, metaOptimizationProblem);
43
44      //Console.WriteLine("Press enter to start");
45      //Console.ReadLine();
46      //TestConfiguration(algorithmVc, baseLevelAlgorithm);
47     
48      //Console.WriteLine("Press enter to start");
49      //Console.ReadLine();
50      TestOptimization(metaLevelAlgorithm);
51
52      //TestMemoryLeak(metaLevelAlgorithm);
53
54      Console.ReadLine();
55    }
56
57    private static void TestMemoryLeak(GeneticAlgorithm metaLevelAlgorithm) {
58      IValueConfiguration algorithmVc = ((MetaOptimizationProblem)metaLevelAlgorithm.Problem).AlgorithmParameterConfiguration;
59
60      Console.WriteLine("Starting Memory Test...");
61      Console.ReadLine();
62
63      var clones = new List<object>();
64      for (int i = 0; i < 1000; i++) {
65        var clone = algorithmVc.Clone();
66        clones.Add(clone);
67      }
68
69      Console.WriteLine("Finished. Now GC...");
70      Console.ReadLine();
71
72      GC.Collect();
73
74      Console.WriteLine("Finished!");
75      Console.ReadLine();
76    }
77
78    private static GeneticAlgorithm GetMetaGA(MetaOptimizationProblem metaOptimizationProblem) {
79      GeneticAlgorithm metaLevelAlgorithm = new GeneticAlgorithm();
80      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
81      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;
82
83      metaLevelAlgorithm.Problem = metaOptimizationProblem;
84      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();
85
86      metaLevelAlgorithm.Mutator = new ParameterConfigurationManipulator();
87      metaLevelAlgorithm.MutationProbability.Value = 0.15;
88
89      return metaLevelAlgorithm;
90    }
91
92    private static EvolutionStrategy GetMetaES(MetaOptimizationProblem metaOptimizationProblem) {
93      EvolutionStrategy metaLevelAlgorithm = new EvolutionStrategy();
94      metaLevelAlgorithm.PopulationSize.Value = metaAlgorithmPopulationSize;
95      metaLevelAlgorithm.MaximumGenerations.Value = metaAlgorithmMaxGenerations;
96
97      metaLevelAlgorithm.Problem = metaOptimizationProblem;
98      metaLevelAlgorithm.Engine = new SequentialEngine.SequentialEngine();
99
100      metaLevelAlgorithm.Mutator = new ParameterConfigurationManipulator();
101      //metaLevelAlgorithm.MutationProbability.Value = 0.15;
102
103      return metaLevelAlgorithm;
104    }
105
106    private static IValueConfiguration SetupGAAlgorithm(GeneticAlgorithm baseLevelAlgorithm, MetaOptimizationProblem metaOptimizationProblem) {
107      baseLevelAlgorithm.Problem = new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem();
108      baseLevelAlgorithm.MaximumGenerations.Value = baseAlgorithmMaxGenerations;
109
110      metaOptimizationProblem.Algorithm = baseLevelAlgorithm;
111      IValueConfiguration algorithmVc = metaOptimizationProblem.AlgorithmParameterConfiguration;
112
113      metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
114        Evaluator = new GriewankEvaluator(),
115        ProblemSize = new IntValue(500)
116      });
117      metaOptimizationProblem.Problems.Add(new HeuristicLab.Problems.TestFunctions.SingleObjectiveTestFunctionProblem() {
118        Evaluator = new GriewankEvaluator(),
119        ProblemSize = new IntValue(1000)
120      });
121
122      ConfigurePopulationSize(algorithmVc);
123      ConfigureMutationRate(algorithmVc);
124      ConfigureMutationOperator(algorithmVc);
125      ConfigureElites(algorithmVc);
126      ConfigureSelectionOperator(algorithmVc);
127      return algorithmVc;
128    }
129
130    private static void TestConfiguration(IValueConfiguration algorithmVc, GeneticAlgorithm baseLevelAlgorithm) {
131      IRandom rand = new FastRandom(0);
132      // set random values
133      for (int i = 0; i < 10; i++) {
134        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
135        clonedVc.Randomize(rand);
136        clonedVc.Parameterize((GeneticAlgorithm)clonedVc.ActualValue.Value);
137        GeneticAlgorithm newAlg = (GeneticAlgorithm)clonedVc.ActualValue.Value;
138        Console.WriteLine(string.Format("PopSize: original: {0}, randomized: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
139        Console.WriteLine(string.Format("MutRate: original: {0}, randomized: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
140        Console.WriteLine(string.Format("MutOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
141        Console.WriteLine(string.Format("SelOp: original: {0}, randomized: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
142        Console.WriteLine(string.Format("GrSi: original: {0}, randomized: {1}", "?", ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
143        Console.WriteLine("---");
144      }
145
146      Console.WriteLine("=======================");
147      algorithmVc.Randomize(rand);
148      algorithmVc.Parameterize(baseLevelAlgorithm);
149      // mutate
150      for (int i = 0; i < 10; i++) {
151        IValueConfiguration clonedVc = (IValueConfiguration)algorithmVc.Clone();
152        clonedVc.Mutate(rand);
153        clonedVc.Parameterize((GeneticAlgorithm)clonedVc.ActualValue.Value);
154        GeneticAlgorithm newAlg = (GeneticAlgorithm)clonedVc.ActualValue.Value;
155        Console.WriteLine(string.Format("PopSize: original: {0}, mutated: {1}", baseLevelAlgorithm.PopulationSize, newAlg.PopulationSize));
156        Console.WriteLine(string.Format("MutRate: original: {0}, mutated: {1}", baseLevelAlgorithm.MutationProbability, newAlg.MutationProbability));
157        Console.WriteLine(string.Format("MutOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Mutator, newAlg.Mutator));
158        Console.WriteLine(string.Format("SelOp: original: {0}, mutated: {1}", baseLevelAlgorithm.Selector, newAlg.Selector));
159        Console.WriteLine(string.Format("GrSi: original: {0}, mutated: {1}", ((TournamentSelector)baseLevelAlgorithm.Selector).GroupSizeParameter.Value, ((TournamentSelector)newAlg.Selector).GroupSizeParameter.Value));
160        Console.WriteLine("---");
161      }
162
163      Console.WriteLine("=======================");
164      // cross
165      for (int i = 0; i < 10; i++) {
166        IValueConfiguration clonedVc1 = (IValueConfiguration)algorithmVc.Clone();
167        IValueConfiguration clonedVc2 = (IValueConfiguration)algorithmVc.Clone();
168
169        GeneticAlgorithm first = (GeneticAlgorithm)clonedVc1.ActualValue.Value.Clone();
170        GeneticAlgorithm second = (GeneticAlgorithm)clonedVc2.ActualValue.Value.Clone();
171
172        clonedVc1.Randomize(rand);
173        clonedVc1.Parameterize(first);
174
175        clonedVc2.Randomize(rand);
176        clonedVc2.Parameterize(second);
177
178        var popSizeBefore = first.PopulationSize.Value;
179        var mutRateBefore = first.MutationProbability.Value;
180        var mutOpBefore = first.Mutator;
181        var selOpBefore = first.Selector;
182        var groupSizeBefore = ((TournamentSelector)first.Selector).GroupSizeParameter.Value.Value;
183
184        clonedVc1.Cross(clonedVc2, rand);
185        clonedVc1.Parameterize(first);
186
187        Console.WriteLine(string.Format("PopSize: first: {0}, second: {1}, crossed: {2}", popSizeBefore, second.PopulationSize, first.PopulationSize));
188        Console.WriteLine(string.Format("MutRate: first: {0}, second: {1}, crossed: {2}", mutRateBefore, second.MutationProbability, first.MutationProbability));
189        Console.WriteLine(string.Format("MutOp: first: {0}, second: {1}, crossed: {2}", mutOpBefore, second.Mutator, first.Mutator));
190        Console.WriteLine(string.Format("SelOp: first: {0}, second: {1}, crossed: {2}", selOpBefore, second.Selector, first.Selector));
191        Console.WriteLine(string.Format("GrSi: first: {0}, second: {1}, crossed: {2}", groupSizeBefore, ((TournamentSelector)second.Selector).GroupSizeParameter.Value, ((TournamentSelector)first.Selector).GroupSizeParameter.Value));
192        Console.WriteLine("---");
193      }
194      Console.WriteLine("=======================");
195    }
196
197    private static void ConfigureMutationOperator(IValueConfiguration algorithmVc) {
198      var mutationOperator = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Mutator").SingleOrDefault();
199      mutationOperator.Optimize = true;
200
201      // uncheck multiMutator to avoid Michalewicz issue
202      var multiMutator = mutationOperator.ValueConfigurations.Where(x => x.ActualValue.Value.ItemName.StartsWith("Multi")).SingleOrDefault();
203      if (multiMutator != null) {
204        mutationOperator.ValueConfigurations.SetItemCheckedState(multiMutator, false);
205      }
206    }
207
208    private static void ConfigureSelectionOperator(IValueConfiguration algorithmVc) {
209      var selectionOperatorPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Selector").SingleOrDefault();
210      selectionOperatorPc.Optimize = true;
211
212      foreach (var vc in selectionOperatorPc.ValueConfigurations) {
213        if (vc.ActualValue.ValueDataType == typeof(TournamentSelector)) {
214          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
215          vc.Optimize = true;
216          ConfigureTournamentGroupSize(vc);
217        } else if (vc.ActualValue.ValueDataType == typeof(RandomSelector)) {
218          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
219        } else {
220          selectionOperatorPc.ValueConfigurations.SetItemCheckedState(vc, true);
221        }
222      }
223    }
224
225    private static void ConfigureTournamentGroupSize(IValueConfiguration tournamentVc) {
226      var groupSizePc = tournamentVc.ParameterConfigurations.Where(x => x.ParameterName == "GroupSize").SingleOrDefault();
227      groupSizePc.Optimize = true;
228
229      groupSizePc.ValueConfigurations.First().Optimize = true;
230      groupSizePc.ValueConfigurations.First().RangeConstraint.LowerBound = new IntValue(0);
231      groupSizePc.ValueConfigurations.First().RangeConstraint.UpperBound = new IntValue(100);
232      groupSizePc.ValueConfigurations.First().RangeConstraint.StepSize = new IntValue(1);
233    }
234
235    private static void ConfigurePopulationSize(IValueConfiguration algorithmVc) {
236      var populationSizePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "PopulationSize").SingleOrDefault();
237      populationSizePc.Optimize = true;
238      var populationSizeVc = populationSizePc.ValueConfigurations.First();
239      populationSizeVc.Optimize = true;
240      populationSizeVc.RangeConstraint.LowerBound = new IntValue(20);
241      populationSizeVc.RangeConstraint.UpperBound = new IntValue(100);
242      populationSizeVc.RangeConstraint.StepSize = new IntValue(1);
243    }
244
245    private static void ConfigureMutationRate(IValueConfiguration algorithmVc) {
246      var mutationRatePc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "MutationProbability").SingleOrDefault();
247      mutationRatePc.Optimize = true;
248      var mutationRateVc = mutationRatePc.ValueConfigurations.First();
249      mutationRateVc.Optimize = true;
250      mutationRateVc.RangeConstraint.LowerBound = new PercentValue(0.0);
251      mutationRateVc.RangeConstraint.UpperBound = new PercentValue(1.0);
252      mutationRateVc.RangeConstraint.StepSize = new PercentValue(0.01);
253    }
254
255    private static void ConfigureElites(IValueConfiguration algorithmVc) {
256      var elitesPc = algorithmVc.ParameterConfigurations.Where(x => x.Name == "Elites").SingleOrDefault();
257      elitesPc.Optimize = true;
258      var elitesVc = elitesPc.ValueConfigurations.First();
259      elitesVc.Optimize = true;
260      elitesVc.RangeConstraint.LowerBound = new IntValue(0);
261      elitesVc.RangeConstraint.UpperBound = new IntValue(20);
262      elitesVc.RangeConstraint.StepSize = new IntValue(1);
263    }
264
265    private static void TestOptimization(EngineAlgorithm metaLevelAlgorithm) {
266      ContentManager.Initialize(new PersistenceContentManager());
267      string path = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Results");
268      if (!Directory.Exists(path))
269        Directory.CreateDirectory(path);
270      string id = DateTime.Now.ToString("yyyy.MM.dd - HH;mm;ss,ffff");
271      string resultPath = Path.Combine(path, string.Format("{0} - Result.hl", id));
272      string outputPath = Path.Combine(path, string.Format("{0} - Console.txt", id));
273
274
275      using (var sw = new StreamWriter(outputPath)) {
276        sw.AutoFlush = true;
277
278        StringBuilder sb1 = new StringBuilder();
279        sb1.AppendLine(string.Format("Meta.PopulationSize: {0}", metaAlgorithmPopulationSize));
280        sb1.AppendLine(string.Format("Meta.MaxGenerations: {0}", metaAlgorithmMaxGenerations));
281        sb1.AppendLine(string.Format("Meta.Repetitions   : {0}", metaProblemRepetitions));
282        sb1.AppendLine(string.Format("Base.MaxGenerations: {0}", baseAlgorithmMaxGenerations));
283        sw.WriteLine(sb1.ToString());
284        Console.WriteLine(sb1.ToString());
285
286        metaLevelAlgorithm.Start();
287        int i = 0;
288        int currentGeneration = -1;
289        do {
290          Thread.Sleep(500);
291          if (metaLevelAlgorithm.Results.ContainsKey("Generations") && ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value != currentGeneration) {
292            while (metaLevelAlgorithm.Results.Count < 3) Thread.Sleep(100);
293            StringBuilder sb = new StringBuilder();
294            sb.AppendLine(DateTime.Now.ToLongTimeString());
295            sb.AppendLine("=================================");
296
297            foreach (var result in metaLevelAlgorithm.Results) {
298              sb.AppendLine(result.ToString());
299              if (result.Name == "Population") {
300                RunCollection rc = (RunCollection)result.Value;
301                var orderedRuns = rc.OrderBy(x => x.Results["RunsAverageQuality"]);
302
303                sb.AppendLine("Qual.  PoSi MutRa Eli GrSi MutOp");
304                foreach (IRun run in orderedRuns) {
305                  string selector;
306                  if (run.Parameters["Selector"] is TournamentSelector) {
307                    selector = string.Format("{0} ({1})", run.Parameters["Selector"].ToString(), ((TournamentSelector)run.Parameters["Selector"]).GroupSizeParameter.Value.ToString());
308                  } else {
309                    selector = string.Format("{0}", run.Parameters["Selector"].ToString());
310                  }
311
312                  sb.AppendLine(string.Format("{0} {1} {2} {3} {4} {5}",
313                    ((DoubleValue)run.Results["RunsAverageQuality"]).Value.ToString("#0.00").PadLeft(7, ' '),
314                    ((IntValue)run.Parameters["PopulationSize"]).Value.ToString().PadLeft(3, ' ').PadRight(3, ' '),
315                    ((DoubleValue)run.Parameters["MutationProbability"]).Value.ToString("0.00").PadLeft(5, ' '),
316                    ((IntValue)run.Parameters["Elites"]).Value.ToString().PadLeft(3, ' '),
317                    Shorten(selector, 20).PadRight(20, ' '),
318                    run.Parameters["Mutator"].ToString()));
319                }
320              }
321            } // foreach
322            Console.Clear();
323            Console.WriteLine(sb.ToString());
324            sw.WriteLine(sb.ToString());
325            currentGeneration = ((IntValue)metaLevelAlgorithm.Results["Generations"].Value).Value;
326          } // if
327          if (i % 30 == 0) GC.Collect();
328          i++;
329        } while (metaLevelAlgorithm.ExecutionState != ExecutionState.Stopped);
330      }
331
332      Console.WriteLine();
333      Console.WriteLine("Storing...");
334
335      ContentManager.Save((IStorableContent)metaLevelAlgorithm, resultPath, true);
336      Console.WriteLine("Finished");
337    }
338
339    private static void TestShorten() {
340      int n = 8;
341      Console.WriteLine(Shorten("1", n));
342      Console.WriteLine(Shorten("12", n));
343      Console.WriteLine(Shorten("123", n));
344      Console.WriteLine(Shorten("1234", n));
345      Console.WriteLine(Shorten("12345", n));
346      Console.WriteLine(Shorten("123456", n));
347      Console.WriteLine(Shorten("1234567", n));
348      Console.WriteLine(Shorten("12345678", n));
349      Console.WriteLine(Shorten("123456789", n));
350      Console.WriteLine(Shorten("1234567890", n));
351      Console.WriteLine(Shorten("12345678901", n));
352    }
353
354    private static string Shorten(string s, int n) {
355      string placeholder = "..";
356      if (s.Length <= n) return s;
357      int len = n / 2 - placeholder.Length / 2;
358      string start = s.Substring(0, len);
359      string end = s.Substring(s.Length - len, len);
360      return start + placeholder + end;
361    }
362
363    private static void TestIntSampling() {
364      System.Random rand = new System.Random();
365      int lower = 10;
366      int upper = 20;
367      int stepsize = 7;
368      for (int i = 0; i < 100; i++) {
369        int val;
370        do {
371          val = rand.Next(lower / stepsize, upper / stepsize + 1) * stepsize;
372        } while (val < lower || val > upper);
373        Console.WriteLine(val);
374      }
375    }
376
377    private static void TestDoubleSampling() {
378      System.Random rand = new System.Random();
379      double lower = 2;
380      double upper = 3;
381      double stepsize = 0.6;
382      for (int i = 0; i < 100; i++) {
383        double val;
384        do {
385          val = Math.Round((rand.NextDouble() * (upper - lower) + lower) / stepsize, 0) * stepsize;
386        } while (val < lower || val > upper);
387        Console.WriteLine(val);
388      }
389    }
390
391    private static IEnumerable<IItem> GetValidValues(IValueParameter valueParameter) {
392      return ApplicationManager.Manager.GetInstances(valueParameter.DataType).Select(x => (IItem)x).OrderBy(x => x.ItemName);
393    }
394  }
395}
Note: See TracBrowser for help on using the repository browser.