1 | using System;
|
---|
2 | using System.Collections.Generic;
|
---|
3 | using System.Linq;
|
---|
4 | using HeuristicLab.Common;
|
---|
5 | using HeuristicLab.Core;
|
---|
6 | using HeuristicLab.Data;
|
---|
7 | using HeuristicLab.Operators;
|
---|
8 | using HeuristicLab.Optimization;
|
---|
9 | using HeuristicLab.Parameters;
|
---|
10 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
11 |
|
---|
12 | namespace HeuristicLab.Problems.MetaOptimization {
|
---|
13 | [Item("AlgorithmRunsAnalyzer", "")]
|
---|
14 | [StorableClass]
|
---|
15 | public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
|
---|
16 |
|
---|
17 | #region Parameter properties
|
---|
18 | public ILookupParameter<DoubleValue> QualityParameter {
|
---|
19 | get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
|
---|
20 | }
|
---|
21 | public LookupParameter<IntValue> GenerationsParameter {
|
---|
22 | get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
|
---|
23 | }
|
---|
24 | public LookupParameter<IntValue> RepetitionsParameter {
|
---|
25 | get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
|
---|
26 | }
|
---|
27 | public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
|
---|
28 | get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
|
---|
29 | }
|
---|
30 | public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
|
---|
31 | get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
|
---|
32 | }
|
---|
33 | public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
|
---|
34 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
|
---|
35 | }
|
---|
36 | public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
|
---|
37 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
|
---|
38 | }
|
---|
39 | public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
|
---|
40 | get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
|
---|
41 | }
|
---|
42 | public LookupParameter<ResultCollection> ResultsParameter {
|
---|
43 | get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
|
---|
44 | }
|
---|
45 | public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
|
---|
46 | get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
|
---|
47 | }
|
---|
48 | public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
|
---|
49 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
|
---|
50 | }
|
---|
51 | public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
|
---|
52 | get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
|
---|
53 | }
|
---|
54 | public LookupParameter<BoolValue> MaximizationParameter {
|
---|
55 | get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
|
---|
56 | }
|
---|
57 | #endregion
|
---|
58 |
|
---|
59 | [StorableConstructor]
|
---|
60 | protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
|
---|
61 | public AlgorithmRunsAnalyzer()
|
---|
62 | : base() {
|
---|
63 | Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
|
---|
64 | Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
|
---|
65 | Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
|
---|
66 | Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
|
---|
67 | Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
|
---|
68 | Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
|
---|
69 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
|
---|
70 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
|
---|
71 | Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
|
---|
72 | Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
|
---|
73 | Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
|
---|
74 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
|
---|
75 | Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
|
---|
76 | Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
|
---|
77 | }
|
---|
78 | protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
|
---|
79 | : base(original, cloner) {
|
---|
80 | }
|
---|
81 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
82 | return new AlgorithmRunsAnalyzer(this, cloner);
|
---|
83 | }
|
---|
84 |
|
---|
85 | public override IOperation Apply() {
|
---|
86 | ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
|
---|
87 | ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
|
---|
88 | ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
|
---|
89 | ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
|
---|
90 | IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
|
---|
91 | IItemList<IProblem> problems = ProblemsParameter.ActualValue;
|
---|
92 | bool maximization = MaximizationParameter.ActualValue.Value;
|
---|
93 | int repetitions = RepetitionsParameter.ActualValue.Value;
|
---|
94 | var resultNames = new List<string> { "BestQuality", "Execution Time", "EvaluatedSolutions" };
|
---|
95 | int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
|
---|
96 | double[] referenceQualityAverages;
|
---|
97 | double[] referenceQualityDeviations;
|
---|
98 | double[] referenceEvaluatedSolutionAverages;
|
---|
99 | GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
|
---|
100 |
|
---|
101 | ResultCollection results = ResultsParameter.ActualValue;
|
---|
102 |
|
---|
103 | if (algorithms.All(x => x.Runs.Count == 1)) {
|
---|
104 | var runs = new RunCollection();
|
---|
105 | var qualities = new double[problems.Count][];
|
---|
106 | var executionTimes = new TimeSpan[problems.Count][];
|
---|
107 | var evaluatedSolutions = new int[problems.Count][];
|
---|
108 |
|
---|
109 | for (int i = 0; i < problems.Count; i++) {
|
---|
110 | qualities[i] = new double[repetitions];
|
---|
111 | evaluatedSolutions[i] = new int[repetitions];
|
---|
112 | executionTimes[i] = new TimeSpan[repetitions];
|
---|
113 | }
|
---|
114 |
|
---|
115 | for (int i = 0; i < algorithms.Length; i++) {
|
---|
116 | int problemIndex = problemIndices[i].Value;
|
---|
117 | int repetitionIndex = repetitionIndices[i].Value;
|
---|
118 |
|
---|
119 | IRun run = algorithms[i].Runs.Single();
|
---|
120 | MetaOptimizationUtil.ClearResults(run, resultNames);
|
---|
121 | MetaOptimizationUtil.ClearParameters(run, parameterNames);
|
---|
122 | run.Results.Add("Meta.FromCache", new BoolValue(false));
|
---|
123 | run.Results.Add("Meta.Generation", new IntValue(currentGeneration));
|
---|
124 | run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
|
---|
125 | run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
|
---|
126 | qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
|
---|
127 | executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
|
---|
128 | evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
|
---|
129 | runs.Add((IRun)run.Clone());
|
---|
130 | }
|
---|
131 |
|
---|
132 | parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
|
---|
133 | parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
|
---|
134 | parameterConfiguration.Repetitions = new IntValue(repetitions);
|
---|
135 | parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
|
---|
136 |
|
---|
137 | if (maximization)
|
---|
138 | parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
|
---|
139 | else
|
---|
140 | parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
|
---|
141 |
|
---|
142 | if (maximization)
|
---|
143 | parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
|
---|
144 | else
|
---|
145 | parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
|
---|
146 |
|
---|
147 | parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
|
---|
148 | parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
|
---|
149 | parameterConfiguration.Runs = runs;
|
---|
150 |
|
---|
151 | this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, 1, 0.1, 1, maximization));
|
---|
152 | } else {
|
---|
153 | // something terrible happened -> most probably due to invalid parameters.
|
---|
154 | // penalty with worst quality from latest generation!
|
---|
155 | double penaltyValue;
|
---|
156 | if (maximization)
|
---|
157 | penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualityAverages.Min();
|
---|
158 | else
|
---|
159 | penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualityAverages.Max();
|
---|
160 | this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
|
---|
161 | }
|
---|
162 |
|
---|
163 | return base.Apply();
|
---|
164 | }
|
---|
165 |
|
---|
166 | private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
|
---|
167 | if (ReferenceQualityAveragesParameter.ActualValue == null) {
|
---|
168 | // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
|
---|
169 | referenceQualityAverages = new double[problemsCount];
|
---|
170 | referenceQualityDeviations = new double[problemsCount];
|
---|
171 | referenceEvaluatedSolutionAverages = new double[problemsCount];
|
---|
172 | for (int i = 0; i < referenceQualityAverages.Length; i++) {
|
---|
173 | referenceQualityAverages[i] = 1;
|
---|
174 | referenceQualityDeviations[i] = 1;
|
---|
175 | referenceEvaluatedSolutionAverages[i] = 1;
|
---|
176 | }
|
---|
177 | } else {
|
---|
178 | referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
|
---|
179 | referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
|
---|
180 | referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
|
---|
181 | }
|
---|
182 | }
|
---|
183 | }
|
---|
184 | }
|
---|