1 | using System;
|
---|
2 | using System.Collections.Generic;
|
---|
3 | using System.Linq;
|
---|
4 | using HeuristicLab.Common;
|
---|
5 | using HeuristicLab.Core;
|
---|
6 | using HeuristicLab.Data;
|
---|
7 | using HeuristicLab.Operators;
|
---|
8 | using HeuristicLab.Optimization;
|
---|
9 | using HeuristicLab.Parameters;
|
---|
10 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
11 |
|
---|
12 | namespace HeuristicLab.Problems.MetaOptimization {
|
---|
13 | [Item("AlgorithmRunsAnalyzer", "")]
|
---|
14 | [StorableClass]
|
---|
15 | public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
|
---|
16 |
|
---|
17 | #region Parameter properties
|
---|
18 | public ILookupParameter<DoubleValue> QualityParameter {
|
---|
19 | get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
|
---|
20 | }
|
---|
21 | public LookupParameter<IntValue> GenerationsParameter {
|
---|
22 | get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
|
---|
23 | }
|
---|
24 | public LookupParameter<IntValue> RepetitionsParameter {
|
---|
25 | get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
|
---|
26 | }
|
---|
27 | public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
|
---|
28 | get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
|
---|
29 | }
|
---|
30 | public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
|
---|
31 | get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
|
---|
32 | }
|
---|
33 | public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
|
---|
34 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
|
---|
35 | }
|
---|
36 | public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
|
---|
37 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
|
---|
38 | }
|
---|
39 | public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
|
---|
40 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
|
---|
41 | }
|
---|
42 | public LookupParameter<ResultCollection> ResultsParameter {
|
---|
43 | get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
|
---|
44 | }
|
---|
45 | public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
|
---|
46 | get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
|
---|
47 | }
|
---|
48 | public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
|
---|
49 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
|
---|
50 | }
|
---|
51 | public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
|
---|
52 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
|
---|
53 | }
|
---|
54 | public LookupParameter<BoolValue> MaximizationParameter {
|
---|
55 | get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
|
---|
56 | }
|
---|
57 | public LookupParameter<DoubleValue> QualityWeightParameter {
|
---|
58 | get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.QualityWeightParameterName]; }
|
---|
59 | }
|
---|
60 | public LookupParameter<DoubleValue> StandardDeviationWeightParameter {
|
---|
61 | get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.StandardDeviationWeightParameterName]; }
|
---|
62 | }
|
---|
63 | public LookupParameter<DoubleValue> EvaluatedSolutionsWeightParameter {
|
---|
64 | get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName]; }
|
---|
65 | }
|
---|
66 | #endregion
|
---|
67 |
|
---|
68 | [StorableConstructor]
|
---|
69 | protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
|
---|
70 | public AlgorithmRunsAnalyzer()
|
---|
71 | : base() {
|
---|
72 | Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
|
---|
73 | Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
|
---|
74 | Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
|
---|
75 | Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
|
---|
76 | Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
|
---|
77 | Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
|
---|
78 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
|
---|
79 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
|
---|
80 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
|
---|
81 | Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
|
---|
82 | Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
|
---|
83 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
|
---|
84 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
|
---|
85 | Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
|
---|
86 | Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.QualityWeightParameterName));
|
---|
87 | Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.StandardDeviationWeightParameterName));
|
---|
88 | Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName));
|
---|
89 | }
|
---|
90 | protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
|
---|
91 | : base(original, cloner) {
|
---|
92 | }
|
---|
93 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
94 | return new AlgorithmRunsAnalyzer(this, cloner);
|
---|
95 | }
|
---|
96 |
|
---|
97 | public override IOperation Apply() {
|
---|
98 | ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
|
---|
99 | ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
|
---|
100 | ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
|
---|
101 | ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
|
---|
102 | IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
|
---|
103 | IItemList<IProblem> problems = ProblemsParameter.ActualValue;
|
---|
104 | bool maximization = MaximizationParameter.ActualValue.Value;
|
---|
105 | int repetitions = RepetitionsParameter.ActualValue.Value;
|
---|
106 | double qualityWeight = QualityWeightParameter.ActualValue.Value;
|
---|
107 | double standardDeviationWeight = StandardDeviationWeightParameter.ActualValue.Value;
|
---|
108 | double evaluatedSolutionsWeight = EvaluatedSolutionsWeightParameter.ActualValue.Value;
|
---|
109 | var resultNames = new List<string> { "BestQuality", "Execution Time", "EvaluatedSolutions" };
|
---|
110 | int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
|
---|
111 | double[] referenceQualityAverages;
|
---|
112 | double[] referenceQualityDeviations;
|
---|
113 | double[] referenceEvaluatedSolutionAverages;
|
---|
114 | GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
|
---|
115 |
|
---|
116 | ResultCollection results = ResultsParameter.ActualValue;
|
---|
117 |
|
---|
118 | if (algorithms.All(x => x.Runs.Count == 1)) {
|
---|
119 | var runs = new RunCollection();
|
---|
120 | var qualities = new double[problems.Count][];
|
---|
121 | var executionTimes = new TimeSpan[problems.Count][];
|
---|
122 | var evaluatedSolutions = new int[problems.Count][];
|
---|
123 |
|
---|
124 | for (int i = 0; i < problems.Count; i++) {
|
---|
125 | qualities[i] = new double[repetitions];
|
---|
126 | evaluatedSolutions[i] = new int[repetitions];
|
---|
127 | executionTimes[i] = new TimeSpan[repetitions];
|
---|
128 | }
|
---|
129 |
|
---|
130 | for (int i = 0; i < algorithms.Length; i++) {
|
---|
131 | int problemIndex = problemIndices[i].Value;
|
---|
132 | int repetitionIndex = repetitionIndices[i].Value;
|
---|
133 |
|
---|
134 | IRun run = algorithms[i].Runs.Single();
|
---|
135 | MetaOptimizationUtil.ClearResults(run, resultNames);
|
---|
136 | MetaOptimizationUtil.ClearParameters(run, parameterNames);
|
---|
137 | run.Results.Add("Meta.FromCache", new BoolValue(false));
|
---|
138 | run.Results.Add("Meta.Generation", new IntValue(currentGeneration));
|
---|
139 | run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
|
---|
140 | run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
|
---|
141 | qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
|
---|
142 | executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
|
---|
143 | evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
|
---|
144 | runs.Add((IRun)run.Clone());
|
---|
145 | }
|
---|
146 |
|
---|
147 | parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
|
---|
148 | parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
|
---|
149 | parameterConfiguration.Repetitions = new IntValue(repetitions);
|
---|
150 | parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
|
---|
151 |
|
---|
152 | if (maximization)
|
---|
153 | parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
|
---|
154 | else
|
---|
155 | parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
|
---|
156 |
|
---|
157 | if (maximization)
|
---|
158 | parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
|
---|
159 | else
|
---|
160 | parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
|
---|
161 |
|
---|
162 | parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
|
---|
163 | parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
|
---|
164 | parameterConfiguration.Runs = runs;
|
---|
165 |
|
---|
166 | this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, qualityWeight, standardDeviationWeight, evaluatedSolutionsWeight, maximization));
|
---|
167 | } else {
|
---|
168 | // something terrible happened -> most probably due to invalid parameters.
|
---|
169 | // penalty with worst quality from latest generation!
|
---|
170 | double penaltyValue;
|
---|
171 | if (maximization)
|
---|
172 | penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Min();
|
---|
173 | else
|
---|
174 | penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Max();
|
---|
175 |
|
---|
176 | this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
|
---|
177 | parameterConfiguration.Quality = new DoubleValue(penaltyValue);
|
---|
178 |
|
---|
179 | parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(Enumerable.Repeat(new TimeSpanValue(TimeSpan.Zero), problems.Count));
|
---|
180 | parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
181 | parameterConfiguration.Repetitions = new IntValue(repetitions);
|
---|
182 | parameterConfiguration.AverageQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
183 | parameterConfiguration.BestQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
184 | parameterConfiguration.WorstQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
185 | parameterConfiguration.QualityVariances = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
186 | parameterConfiguration.QualityStandardDeviations = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
|
---|
187 | parameterConfiguration.Runs = null;
|
---|
188 | }
|
---|
189 |
|
---|
190 | return base.Apply();
|
---|
191 | }
|
---|
192 |
|
---|
193 | private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
|
---|
194 | if (ReferenceQualityAveragesParameter.ActualValue == null) {
|
---|
195 | // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
|
---|
196 | referenceQualityAverages = new double[problemsCount];
|
---|
197 | referenceQualityDeviations = new double[problemsCount];
|
---|
198 | referenceEvaluatedSolutionAverages = new double[problemsCount];
|
---|
199 | for (int i = 0; i < referenceQualityAverages.Length; i++) {
|
---|
200 | referenceQualityAverages[i] = 1;
|
---|
201 | referenceQualityDeviations[i] = 1;
|
---|
202 | referenceEvaluatedSolutionAverages[i] = 1;
|
---|
203 | }
|
---|
204 | } else {
|
---|
205 | referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
|
---|
206 | referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
|
---|
207 | referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
|
---|
208 | }
|
---|
209 | }
|
---|
210 | }
|
---|
211 | }
|
---|