Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.MetaOptimization (trunk integration)/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs @ 8576

Last change on this file since 8576 was 8576, checked in by jkarder, 11 years ago

#1853: created branch for MetaOptimization (trunk integration)
File size: 16.5 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using HeuristicLab.Common;
26using HeuristicLab.Core;
27using HeuristicLab.Data;
28using HeuristicLab.Encodings.ParameterConfigurationEncoding;
29using HeuristicLab.Operators;
30using HeuristicLab.Optimization;
31using HeuristicLab.Parameters;
32using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
33
34namespace HeuristicLab.Problems.MetaOptimization {
35  /// <summary>
36  /// TODO
37  /// </summary>
38  [Item("AlgorithmRunsAnalyzer", "TODO")]
39  [StorableClass]
40  public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
41    #region Parameter properties
42    public ILookupParameter<DoubleValue> QualityParameter {
43      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
44    }
45    public LookupParameter<IntValue> GenerationsParameter {
46      get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
47    }
48    public LookupParameter<IntValue> RepetitionsParameter {
49      get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
50    }
51    public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
52      get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
53    }
54    public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
55      get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
56    }
57    public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
58      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
59    }
60    public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
61      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
62    }
63    public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
64      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
65    }
66    public LookupParameter<ResultCollection> ResultsParameter {
67      get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
68    }
69    public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
70      get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
71    }
72    public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
73      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
74    }
75    public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
76      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
77    }
78    public LookupParameter<BoolValue> MaximizationParameter {
79      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
80    }
81    public LookupParameter<DoubleValue> QualityWeightParameter {
82      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.QualityWeightParameterName]; }
83    }
84    public LookupParameter<DoubleValue> StandardDeviationWeightParameter {
85      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.StandardDeviationWeightParameterName]; }
86    }
87    public LookupParameter<DoubleValue> EvaluatedSolutionsWeightParameter {
88      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName]; }
89    }
90    private ScopeParameter CurrentScopeParameter {
91      get { return (ScopeParameter)Parameters["CurrentScope"]; }
92    }
93    public IScope CurrentScope {
94      get { return CurrentScopeParameter.ActualValue; }
95    }
96    public LookupParameter<StringValue> QualityMeasureNameParameter {
97      get { return (LookupParameter<StringValue>)Parameters[MetaOptimizationProblem.QualityMeasureNameName]; }
98    }
99    #endregion
100
101    #region Constructors and Cloning
102    public AlgorithmRunsAnalyzer()
103      : base() {
104      Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
105      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
106      Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
107      Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
108      Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
109      Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
110      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
111      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
112      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
113      Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
114      Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
115      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
116      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
117      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
118      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.QualityWeightParameterName));
119      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.StandardDeviationWeightParameterName));
120      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName));
121      Parameters.Add(new LookupParameter<StringValue>(MetaOptimizationProblem.QualityMeasureNameName));
122      Parameters.Add(new ScopeParameter("CurrentScope", "The current scope whose sub-scopes represent the parents."));
123    }
124
125    [StorableConstructor]
126    protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
127    protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner) : base(original, cloner) { }
128    public override IDeepCloneable Clone(Cloner cloner) {
129      return new AlgorithmRunsAnalyzer(this, cloner);
130    }
131    [StorableHook(HookType.AfterDeserialization)]
132    private void AfterDeserialization() {
133      if (!Parameters.ContainsKey("CurrentScope")) Parameters.Add(new ScopeParameter("CurrentScope", "The current scope whose sub-scopes represent the parents.")); // backwards compatibility
134      if (!Parameters.ContainsKey(MetaOptimizationProblem.QualityMeasureNameName)) Parameters.Add(new LookupParameter<StringValue>(MetaOptimizationProblem.QualityMeasureNameName)); // backwards compatibility
135    }
136    #endregion
137
138    public override IOperation Apply() {
139      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
140      ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
141      ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
142      ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
143      IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
144      IItemList<IProblem> problems = ProblemsParameter.ActualValue;
145      bool maximization = MaximizationParameter.ActualValue.Value;
146      int repetitions = RepetitionsParameter.ActualValue.Value;
147      double qualityWeight = QualityWeightParameter.ActualValue.Value;
148      double standardDeviationWeight = StandardDeviationWeightParameter.ActualValue.Value;
149      double evaluatedSolutionsWeight = EvaluatedSolutionsWeightParameter.ActualValue.Value;
150      string qualityMeasureName = QualityMeasureNameParameter.ActualValue.Value;
151      var resultNames = new List<string> { qualityMeasureName, "Execution Time", "EvaluatedSolutions" };
152      int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
153      double[] referenceQualityAverages;
154      double[] referenceQualityDeviations;
155      double[] referenceEvaluatedSolutionAverages;
156      GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
157
158      ResultCollection results = ResultsParameter.ActualValue;
159
160      if (algorithms.All(x => x.Runs.Count == 1)) {
161        var runs = new RunCollection();
162        var qualities = new double[problems.Count][];
163        var executionTimes = new TimeSpan[problems.Count][];
164        var evaluatedSolutions = new int[problems.Count][];
165
166        for (int i = 0; i < problems.Count; i++) {
167          qualities[i] = new double[repetitions];
168          evaluatedSolutions[i] = new int[repetitions];
169          executionTimes[i] = new TimeSpan[repetitions];
170        }
171
172        for (int i = 0; i < algorithms.Length; i++) {
173          int problemIndex = problemIndices[i].Value;
174          int repetitionIndex = repetitionIndices[i].Value;
175
176          IRun run = (IRun)algorithms[i].Runs.Single().Clone();
177          MetaOptimizationUtil.ClearResults(run, resultNames);
178          MetaOptimizationUtil.ClearParameters(run, parameterNames);
179          run.Results.Add("Meta-FromCache", new BoolValue(false));
180          run.Results.Add("Meta-Generation", new IntValue(currentGeneration));
181          run.Results.Add("Meta-ProblemIndex", new IntValue(problemIndex));
182          run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
183
184          qualities[problemIndex][repetitionIndex] = GetResultValue<DoubleValue>(run.Results, qualityMeasureName).Value;
185          executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
186          if (run.Results.ContainsKey("EvaluatedSolutions")) {
187            evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
188          } else {
189            evaluatedSolutions[problemIndex][repetitionIndex] = 1;
190          }
191          runs.Add(run);
192        }
193
194        parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
195        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
196        parameterConfiguration.Repetitions = new IntValue(repetitions);
197        parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
198
199        if (maximization)
200          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
201        else
202          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
203
204        if (maximization)
205          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
206        else
207          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
208
209        parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
210        parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
211        parameterConfiguration.Runs = runs;
212
213        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, qualityWeight, standardDeviationWeight, evaluatedSolutionsWeight, maximization));
214      } else {
215        // something terrible happened -> most probably due to invalid parameters.
216        // penalty with worst quality from latest generation!
217        double penaltyValue;
218        if (maximization)
219          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Min();
220        else
221          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Max();
222
223        this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
224        parameterConfiguration.Quality = new DoubleValue(penaltyValue);
225
226        parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(Enumerable.Repeat(new TimeSpanValue(TimeSpan.Zero), problems.Count));
227        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
228        parameterConfiguration.Repetitions = new IntValue(repetitions);
229        parameterConfiguration.AverageQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
230        parameterConfiguration.BestQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
231        parameterConfiguration.WorstQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
232        parameterConfiguration.QualityVariances = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
233        parameterConfiguration.QualityStandardDeviations = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
234        parameterConfiguration.Runs = null;
235      }
236
237      // in OSGA there are more subscopes, so be careful which to delete
238      CurrentScope.SubScopes.RemoveAll(x => x.Variables.Count((v) => (v.Name == "RepetitionCount")) == 1);
239
240      return base.Apply();
241    }
242
243    private T1 GetResultValue<T1>(IDictionary<string, IItem> results, string resultName) {
244      return (T1)results[resultName];
245
246      //string separator = ".";
247      //string[] tokens = resultName.Split(separator.ToCharArray());
248
249      //IDictionary<string, IItem> currentResults = results;
250      //IItem currentResult = null;
251      //for (int i = 0; i < tokens.Length; i++) {
252      //  if(currentResults == null)
253      //    throw new KeyNotFoundException("Result value " + resultName + " was not found");
254      //  if (currentResults.ContainsKey(tokens[i])) {
255      //    currentResult = currentResults[tokens[i]];
256      //    currentResults = currentResult as IDictionary<string, IItem>;
257      //  } else {
258      //    throw new KeyNotFoundException("Result value " + resultName + " was not found");
259      //  }
260      //}
261      //return (T1)currentResult;
262    }
263
264    private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
265      if (ReferenceQualityAveragesParameter.ActualValue == null) {
266        // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
267        referenceQualityAverages = new double[problemsCount];
268        referenceQualityDeviations = new double[problemsCount];
269        referenceEvaluatedSolutionAverages = new double[problemsCount];
270        for (int i = 0; i < referenceQualityAverages.Length; i++) {
271          referenceQualityAverages[i] = 1;
272          referenceQualityDeviations[i] = 1;
273          referenceEvaluatedSolutionAverages[i] = 1;
274        }
275      } else {
276        referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
277        referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
278        referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
279      }
280    }
281  }
282}
Note: See TracBrowser for help on using the repository browser.