[5653] | 1 | using System;
|
---|
| 2 | using System.Collections.Generic;
|
---|
| 3 | using System.Linq;
|
---|
| 4 | using System.Text;
|
---|
| 5 | using HeuristicLab.Operators;
|
---|
| 6 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
| 7 | using HeuristicLab.Core;
|
---|
| 8 | using HeuristicLab.Common;
|
---|
| 9 | using HeuristicLab.Parameters;
|
---|
| 10 | using HeuristicLab.Data;
|
---|
| 11 | using HeuristicLab.Optimization;
|
---|
| 12 |
|
---|
| 13 | namespace HeuristicLab.Problems.MetaOptimization {
|
---|
| 14 | [Item("AlgorithmRunsAnalyzer", "")]
|
---|
| 15 | [StorableClass]
|
---|
| 16 | public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
|
---|
| 17 |
|
---|
| 18 | #region Parameter properties
|
---|
| 19 | public ILookupParameter<DoubleValue> QualityParameter {
|
---|
| 20 | get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
|
---|
| 21 | }
|
---|
| 22 | public LookupParameter<IntValue> GenerationsParameter {
|
---|
| 23 | get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
|
---|
| 24 | }
|
---|
| 25 | public LookupParameter<IntValue> RepetitionsParameter {
|
---|
| 26 | get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
|
---|
| 27 | }
|
---|
| 28 | public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
|
---|
| 29 | get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
|
---|
| 30 | }
|
---|
| 31 | public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
|
---|
| 32 | get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
|
---|
| 33 | }
|
---|
| 34 | public LookupParameter<DoubleArray> ProblemQualityReferencesParameter {
|
---|
| 35 | get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
|
---|
| 36 | }
|
---|
| 37 | public LookupParameter<ResultCollection> ResultsParameter {
|
---|
| 38 | get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
|
---|
| 39 | }
|
---|
| 40 | public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
|
---|
| 41 | get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
|
---|
| 42 | }
|
---|
| 43 | public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
|
---|
| 44 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
|
---|
| 45 | }
|
---|
| 46 | public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
|
---|
| 47 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
|
---|
| 48 | }
|
---|
| 49 | #endregion
|
---|
| 50 |
|
---|
| 51 | [StorableConstructor]
|
---|
| 52 | protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
|
---|
| 53 | public AlgorithmRunsAnalyzer() : base() {
|
---|
| 54 | Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
|
---|
| 55 | Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
|
---|
| 56 | Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
|
---|
| 57 | Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
|
---|
| 58 | Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
|
---|
| 59 | Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
|
---|
| 60 | Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
|
---|
| 61 | Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
|
---|
| 62 | Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
|
---|
| 63 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
|
---|
| 64 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
|
---|
| 65 | }
|
---|
| 66 | protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
|
---|
| 67 | : base(original, cloner) {
|
---|
| 68 | }
|
---|
| 69 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
| 70 | return new AlgorithmRunsAnalyzer(this, cloner);
|
---|
| 71 | }
|
---|
| 72 |
|
---|
| 73 | public override IOperation Apply() {
|
---|
| 74 | ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
|
---|
| 75 | ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
|
---|
| 76 | ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
|
---|
| 77 | ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
|
---|
| 78 | IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
|
---|
| 79 | IItemList<IProblem> problems = ProblemsParameter.ActualValue;
|
---|
| 80 | int repetitions = RepetitionsParameter.ActualValue.Value;
|
---|
| 81 | var resultNames = new List<string> { "BestQuality", "Execution Time" };
|
---|
| 82 | int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
|
---|
| 83 | double[] referenceQualities = GetReferenceQualities(problems.Count);
|
---|
| 84 | ResultCollection results = ResultsParameter.ActualValue;
|
---|
| 85 |
|
---|
| 86 | if (algorithms.All(x => x.Runs.Count == 1)) {
|
---|
| 87 | var runs = new RunCollection();
|
---|
| 88 | var qualities = new double[problems.Count][];
|
---|
| 89 | var executionTimes = new TimeSpan[problems.Count][];
|
---|
| 90 |
|
---|
| 91 | for (int i = 0; i < problems.Count; i++) {
|
---|
| 92 | qualities[i] = new double[repetitions];
|
---|
| 93 | executionTimes[i] = new TimeSpan[repetitions];
|
---|
| 94 | }
|
---|
| 95 |
|
---|
| 96 | for (int i = 0; i < algorithms.Length; i++) {
|
---|
| 97 | int problemIndex = problemIndices[i].Value;
|
---|
| 98 | int repetitionIndex = repetitionIndices[i].Value;
|
---|
| 99 |
|
---|
| 100 | IRun run = algorithms[i].Runs.First();
|
---|
| 101 | MetaOptimizationUtil.ClearResults(run, resultNames);
|
---|
| 102 | MetaOptimizationUtil.ClearParameters(run, parameterNames);
|
---|
| 103 | run.Results.Add("Meta.FromCache", new BoolValue(false));
|
---|
| 104 | run.Results.Add("Meta.Generation", new IntValue(currentGeneration));
|
---|
| 105 | run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
|
---|
| 106 | run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
|
---|
| 107 | qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
|
---|
| 108 | executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
|
---|
| 109 | runs.Add((IRun)run.Clone());
|
---|
| 110 | }
|
---|
| 111 |
|
---|
| 112 | parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
|
---|
| 113 | parameterConfiguration.Repetitions = new IntValue(repetitions);
|
---|
| 114 | parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray()); // todo: respect Maximization:true/false
|
---|
| 115 | parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
|
---|
| 116 | parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray()); // todo: respect Maximization:true/false
|
---|
| 117 | parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
|
---|
| 118 | parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
|
---|
| 119 | parameterConfiguration.Runs = runs;
|
---|
| 120 |
|
---|
| 121 | this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.NormalizeQualities(parameterConfiguration, referenceQualities));
|
---|
| 122 | } else {
|
---|
| 123 | // something terrible happened -> most probably due to invalid parameters.
|
---|
| 124 | // penalty with worst quality from latest generation!
|
---|
| 125 |
|
---|
| 126 | double penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualities.Max(); // todo: respect min/max
|
---|
| 127 | this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
|
---|
| 128 | }
|
---|
| 129 |
|
---|
| 130 | return base.Apply();
|
---|
| 131 | }
|
---|
| 132 |
|
---|
| 133 | private double[] GetReferenceQualities(int problemsCount) {
|
---|
| 134 | double[] referenceQualities;
|
---|
| 135 | if (ProblemQualityReferencesParameter.ActualValue == null) {
|
---|
| 136 | // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
|
---|
| 137 | referenceQualities = new double[problemsCount];
|
---|
| 138 | for (int i = 0; i < referenceQualities.Length; i++) {
|
---|
| 139 | referenceQualities[i] = 1;
|
---|
| 140 | }
|
---|
| 141 | } else {
|
---|
| 142 | referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
|
---|
| 143 | }
|
---|
| 144 | return referenceQualities;
|
---|
| 145 | }
|
---|
| 146 | }
|
---|
| 147 | }
|
---|