1 | using System;
|
---|
2 | using System.Collections.Generic;
|
---|
3 | using System.Linq;
|
---|
4 | using HeuristicLab.Common;
|
---|
5 | using HeuristicLab.Core;
|
---|
6 | using HeuristicLab.Data;
|
---|
7 | using HeuristicLab.Operators;
|
---|
8 | using HeuristicLab.Optimization;
|
---|
9 | using HeuristicLab.Parameters;
|
---|
10 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
11 |
|
---|
12 | namespace HeuristicLab.Problems.MetaOptimization {
|
---|
13 | [Item("AlgorithmRunsAnalyzer", "")]
|
---|
14 | [StorableClass]
|
---|
15 | public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
|
---|
16 |
|
---|
17 | #region Parameter properties
|
---|
18 | public ILookupParameter<DoubleValue> QualityParameter {
|
---|
19 | get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
|
---|
20 | }
|
---|
21 | public LookupParameter<IntValue> GenerationsParameter {
|
---|
22 | get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
|
---|
23 | }
|
---|
24 | public LookupParameter<IntValue> RepetitionsParameter {
|
---|
25 | get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
|
---|
26 | }
|
---|
27 | public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
|
---|
28 | get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
|
---|
29 | }
|
---|
30 | public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
|
---|
31 | get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
|
---|
32 | }
|
---|
33 | public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
|
---|
34 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
|
---|
35 | }
|
---|
36 | public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
|
---|
37 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
|
---|
38 | }
|
---|
39 | public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
|
---|
40 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
|
---|
41 | }
|
---|
42 | public LookupParameter<ResultCollection> ResultsParameter {
|
---|
43 | get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
|
---|
44 | }
|
---|
45 | public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
|
---|
46 | get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
|
---|
47 | }
|
---|
48 | public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
|
---|
49 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
|
---|
50 | }
|
---|
51 | public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
|
---|
52 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
|
---|
53 | }
|
---|
54 | public LookupParameter<BoolValue> MaximizationParameter {
|
---|
55 | get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
|
---|
56 | }
|
---|
57 | public LookupParameter<DoubleValue> QualityWeightParameter {
|
---|
58 | get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.QualityWeightParameterName]; }
|
---|
59 | }
|
---|
60 | public LookupParameter<DoubleValue> StandardDeviationWeightParameter {
|
---|
61 | get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.StandardDeviationWeightParameterName]; }
|
---|
62 | }
|
---|
63 | public LookupParameter<DoubleValue> EvaluatedSolutionsWeightParameter {
|
---|
64 | get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName]; }
|
---|
65 | }
|
---|
66 | private ScopeParameter CurrentScopeParameter {
|
---|
67 | get { return (ScopeParameter)Parameters["CurrentScope"]; }
|
---|
68 | }
|
---|
69 | public IScope CurrentScope {
|
---|
70 | get { return CurrentScopeParameter.ActualValue; }
|
---|
71 | }
|
---|
72 | #endregion
|
---|
73 |
|
---|
74 | [StorableConstructor]
|
---|
75 | protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
|
---|
76 | public AlgorithmRunsAnalyzer()
|
---|
77 | : base() {
|
---|
78 | Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
|
---|
79 | Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
|
---|
80 | Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
|
---|
81 | Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
|
---|
82 | Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
|
---|
83 | Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
|
---|
84 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
|
---|
85 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
|
---|
86 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
|
---|
87 | Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
|
---|
88 | Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
|
---|
89 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
|
---|
90 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
|
---|
91 | Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
|
---|
92 | Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.QualityWeightParameterName));
|
---|
93 | Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.StandardDeviationWeightParameterName));
|
---|
94 | Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName));
|
---|
95 | Parameters.Add(new ScopeParameter("CurrentScope", "The current scope whose sub-scopes represent the parents."));
|
---|
96 | }
|
---|
97 | protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
|
---|
98 | : base(original, cloner) {
|
---|
99 | }
|
---|
100 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
101 | return new AlgorithmRunsAnalyzer(this, cloner);
|
---|
102 | }
|
---|
103 | [StorableHook(HookType.AfterDeserialization)]
|
---|
104 | private void AfterDeserialization() {
|
---|
105 | if (!Parameters.ContainsKey("CurrentScope")) Parameters.Add(new ScopeParameter("CurrentScope", "The current scope whose sub-scopes represent the parents.")); // backwards compatibility
|
---|
106 | }
|
---|
107 |
|
---|
108 | public override IOperation Apply() {
|
---|
109 | ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
|
---|
110 | ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
|
---|
111 | ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
|
---|
112 | ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
|
---|
113 | IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
|
---|
114 | IItemList<IProblem> problems = ProblemsParameter.ActualValue;
|
---|
115 | bool maximization = MaximizationParameter.ActualValue.Value;
|
---|
116 | int repetitions = RepetitionsParameter.ActualValue.Value;
|
---|
117 | double qualityWeight = QualityWeightParameter.ActualValue.Value;
|
---|
118 | double standardDeviationWeight = StandardDeviationWeightParameter.ActualValue.Value;
|
---|
119 | double evaluatedSolutionsWeight = EvaluatedSolutionsWeightParameter.ActualValue.Value;
|
---|
120 | var resultNames = new List<string> { "BestQuality", "Execution Time", "EvaluatedSolutions" };
|
---|
121 | int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
|
---|
122 | double[] referenceQualityAverages;
|
---|
123 | double[] referenceQualityDeviations;
|
---|
124 | double[] referenceEvaluatedSolutionAverages;
|
---|
125 | GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
|
---|
126 |
|
---|
127 | ResultCollection results = ResultsParameter.ActualValue;
|
---|
128 |
|
---|
129 | if (algorithms.All(x => x.Runs.Count == 1)) {
|
---|
130 | var runs = new RunCollection();
|
---|
131 | var qualities = new double[problems.Count][];
|
---|
132 | var executionTimes = new TimeSpan[problems.Count][];
|
---|
133 | var evaluatedSolutions = new int[problems.Count][];
|
---|
134 |
|
---|
135 | for (int i = 0; i < problems.Count; i++) {
|
---|
136 | qualities[i] = new double[repetitions];
|
---|
137 | evaluatedSolutions[i] = new int[repetitions];
|
---|
138 | executionTimes[i] = new TimeSpan[repetitions];
|
---|
139 | }
|
---|
140 |
|
---|
141 | for (int i = 0; i < algorithms.Length; i++) {
|
---|
142 | int problemIndex = problemIndices[i].Value;
|
---|
143 | int repetitionIndex = repetitionIndices[i].Value;
|
---|
144 |
|
---|
145 | IRun run = (IRun)algorithms[i].Runs.Single().Clone();
|
---|
146 | MetaOptimizationUtil.ClearResults(run, resultNames);
|
---|
147 | MetaOptimizationUtil.ClearParameters(run, parameterNames);
|
---|
148 | run.Results.Add("Meta.FromCache", new BoolValue(false));
|
---|
149 | run.Results.Add("Meta.Generation", new IntValue(currentGeneration));
|
---|
150 | run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
|
---|
151 | run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
|
---|
152 | qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
|
---|
153 | executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
|
---|
154 | evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
|
---|
155 | runs.Add(run);
|
---|
156 | }
|
---|
157 |
|
---|
158 | parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
|
---|
159 | parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
|
---|
160 | parameterConfiguration.Repetitions = new IntValue(repetitions);
|
---|
161 | parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
|
---|
162 |
|
---|
163 | if (maximization)
|
---|
164 | parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
|
---|
165 | else
|
---|
166 | parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
|
---|
167 |
|
---|
168 | if (maximization)
|
---|
169 | parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
|
---|
170 | else
|
---|
171 | parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
|
---|
172 |
|
---|
173 | parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
|
---|
174 | parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
|
---|
175 | parameterConfiguration.Runs = runs;
|
---|
176 |
|
---|
177 | this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, qualityWeight, standardDeviationWeight, evaluatedSolutionsWeight, maximization));
|
---|
178 | } else {
|
---|
179 | // something terrible happened -> most probably due to invalid parameters.
|
---|
180 | // penalty with worst quality from latest generation!
|
---|
181 | double penaltyValue;
|
---|
182 | if (maximization)
|
---|
183 | penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Min();
|
---|
184 | else
|
---|
185 | penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Max();
|
---|
186 |
|
---|
187 | this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
|
---|
188 | parameterConfiguration.Quality = new DoubleValue(penaltyValue);
|
---|
189 |
|
---|
190 | parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(Enumerable.Repeat(new TimeSpanValue(TimeSpan.Zero), problems.Count));
|
---|
191 | parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
192 | parameterConfiguration.Repetitions = new IntValue(repetitions);
|
---|
193 | parameterConfiguration.AverageQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
194 | parameterConfiguration.BestQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
195 | parameterConfiguration.WorstQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
196 | parameterConfiguration.QualityVariances = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
197 | parameterConfiguration.QualityStandardDeviations = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
198 | parameterConfiguration.Runs = null;
|
---|
199 | }
|
---|
200 |
|
---|
201 | // in OSGA there are more subscopes, so be careful which to delete
|
---|
202 | CurrentScope.SubScopes.RemoveAll(x => x.Variables.Count((v) => (v.Name == "RepetitionCount")) == 1);
|
---|
203 |
|
---|
204 | return base.Apply();
|
---|
205 | }
|
---|
206 |
|
---|
207 | private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
|
---|
208 | if (ReferenceQualityAveragesParameter.ActualValue == null) {
|
---|
209 | // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
|
---|
210 | referenceQualityAverages = new double[problemsCount];
|
---|
211 | referenceQualityDeviations = new double[problemsCount];
|
---|
212 | referenceEvaluatedSolutionAverages = new double[problemsCount];
|
---|
213 | for (int i = 0; i < referenceQualityAverages.Length; i++) {
|
---|
214 | referenceQualityAverages[i] = 1;
|
---|
215 | referenceQualityDeviations[i] = 1;
|
---|
216 | referenceEvaluatedSolutionAverages[i] = 1;
|
---|
217 | }
|
---|
218 | } else {
|
---|
219 | referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
|
---|
220 | referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
|
---|
221 | referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
|
---|
222 | }
|
---|
223 | }
|
---|
224 | }
|
---|
225 | }
|
---|