Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs @ 14777

Last change on this file since 14777 was 7253, checked in by gkronber, 13 years ago

#1727 set the number of evaluated solutions to 1 if the result is missing in the algorithm.
File size: 15.5 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Linq;
4using HeuristicLab.Common;
5using HeuristicLab.Core;
6using HeuristicLab.Data;
7using HeuristicLab.Operators;
8using HeuristicLab.Optimization;
9using HeuristicLab.Parameters;
10using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
11
12namespace HeuristicLab.Problems.MetaOptimization {
13  [Item("AlgorithmRunsAnalyzer", "")]
14  [StorableClass]
15  public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
16
17    #region Parameter properties
18    public ILookupParameter<DoubleValue> QualityParameter {
19      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
20    }
21    public LookupParameter<IntValue> GenerationsParameter {
22      get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
23    }
24    public LookupParameter<IntValue> RepetitionsParameter {
25      get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
26    }
27    public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
28      get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
29    }
30    public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
31      get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
32    }
33    public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
34      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
35    }
36    public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
37      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
38    }
39    public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
40      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
41    }
42    public LookupParameter<ResultCollection> ResultsParameter {
43      get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
44    }
45    public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
46      get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
47    }
48    public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
49      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
50    }
51    public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
52      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
53    }
54    public LookupParameter<BoolValue> MaximizationParameter {
55      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
56    }
57    public LookupParameter<DoubleValue> QualityWeightParameter {
58      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.QualityWeightParameterName]; }
59    }
60    public LookupParameter<DoubleValue> StandardDeviationWeightParameter {
61      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.StandardDeviationWeightParameterName]; }
62    }
63    public LookupParameter<DoubleValue> EvaluatedSolutionsWeightParameter {
64      get { return (LookupParameter<DoubleValue>)Parameters[MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName]; }
65    }
66    private ScopeParameter CurrentScopeParameter {
67      get { return (ScopeParameter)Parameters["CurrentScope"]; }
68    }
69    public IScope CurrentScope {
70      get { return CurrentScopeParameter.ActualValue; }
71    }
72    public LookupParameter<StringValue> QualityMeasureNameParameter {
73      get { return (LookupParameter<StringValue>)Parameters[MetaOptimizationProblem.QualityMeasureNameName]; }
74    }
75    #endregion
76
77    [StorableConstructor]
78    protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
79    public AlgorithmRunsAnalyzer()
80      : base() {
81      Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
82      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
83      Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
84      Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
85      Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
86      Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
87      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
88      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
89      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
90      Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
91      Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
92      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
93      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
94      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
95      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.QualityWeightParameterName));
96      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.StandardDeviationWeightParameterName));
97      Parameters.Add(new LookupParameter<DoubleValue>(MetaOptimizationProblem.EvaluatedSolutionsWeightParameterName));
98      Parameters.Add(new LookupParameter<StringValue>(MetaOptimizationProblem.QualityMeasureNameName));
99      Parameters.Add(new ScopeParameter("CurrentScope", "The current scope whose sub-scopes represent the parents."));
100    }
101    protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
102      : base(original, cloner) {
103    }
104    public override IDeepCloneable Clone(Cloner cloner) {
105      return new AlgorithmRunsAnalyzer(this, cloner);
106    }
107    [StorableHook(HookType.AfterDeserialization)]
108    private void AfterDeserialization() {
109      if (!Parameters.ContainsKey("CurrentScope")) Parameters.Add(new ScopeParameter("CurrentScope", "The current scope whose sub-scopes represent the parents.")); // backwards compatibility
110      if (!Parameters.ContainsKey(MetaOptimizationProblem.QualityMeasureNameName)) Parameters.Add(new LookupParameter<StringValue>(MetaOptimizationProblem.QualityMeasureNameName)); // backwards compatibility
111    }
112
113    public override IOperation Apply() {
114      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
115      ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
116      ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
117      ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
118      IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
119      IItemList<IProblem> problems = ProblemsParameter.ActualValue;
120      bool maximization = MaximizationParameter.ActualValue.Value;
121      int repetitions = RepetitionsParameter.ActualValue.Value;
122      double qualityWeight = QualityWeightParameter.ActualValue.Value;
123      double standardDeviationWeight = StandardDeviationWeightParameter.ActualValue.Value;
124      double evaluatedSolutionsWeight = EvaluatedSolutionsWeightParameter.ActualValue.Value;
125      string qualityMeasureName = QualityMeasureNameParameter.ActualValue.Value;
126      var resultNames = new List<string> { qualityMeasureName, "Execution Time", "EvaluatedSolutions" };
127      int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
128      double[] referenceQualityAverages;
129      double[] referenceQualityDeviations;
130      double[] referenceEvaluatedSolutionAverages;
131      GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
132
133      ResultCollection results = ResultsParameter.ActualValue;
134
135      if (algorithms.All(x => x.Runs.Count == 1)) {
136        var runs = new RunCollection();
137        var qualities = new double[problems.Count][];
138        var executionTimes = new TimeSpan[problems.Count][];
139        var evaluatedSolutions = new int[problems.Count][];
140
141        for (int i = 0; i < problems.Count; i++) {
142          qualities[i] = new double[repetitions];
143          evaluatedSolutions[i] = new int[repetitions];
144          executionTimes[i] = new TimeSpan[repetitions];
145        }
146
147        for (int i = 0; i < algorithms.Length; i++) {
148          int problemIndex = problemIndices[i].Value;
149          int repetitionIndex = repetitionIndices[i].Value;
150
151          IRun run = (IRun)algorithms[i].Runs.Single().Clone();
152          MetaOptimizationUtil.ClearResults(run, resultNames);
153          MetaOptimizationUtil.ClearParameters(run, parameterNames);
154          run.Results.Add("Meta-FromCache", new BoolValue(false));
155          run.Results.Add("Meta-Generation", new IntValue(currentGeneration));
156          run.Results.Add("Meta-ProblemIndex", new IntValue(problemIndex));
157          run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
158
159          qualities[problemIndex][repetitionIndex] = GetResultValue<DoubleValue>(run.Results, qualityMeasureName).Value;
160          executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
161          if (run.Results.ContainsKey("EvaluatedSolutions")) {
162            evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
163          } else {
164            evaluatedSolutions[problemIndex][repetitionIndex] = 1;
165          }
166          runs.Add(run);
167        }
168
169        parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
170        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
171        parameterConfiguration.Repetitions = new IntValue(repetitions);
172        parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
173
174        if (maximization)
175          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
176        else
177          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
178
179        if (maximization)
180          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
181        else
182          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
183
184        parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
185        parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
186        parameterConfiguration.Runs = runs;
187
188        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, qualityWeight, standardDeviationWeight, evaluatedSolutionsWeight, maximization));
189      } else {
190        // something terrible happened -> most probably due to invalid parameters.
191        // penalty with worst quality from latest generation!
192        double penaltyValue;
193        if (maximization)
194          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Min();
195        else
196          penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"].Value).Value : referenceQualityAverages.Max();
197
198        this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
199        parameterConfiguration.Quality = new DoubleValue(penaltyValue);
200
201        parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(Enumerable.Repeat(new TimeSpanValue(TimeSpan.Zero), problems.Count));
202        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
203        parameterConfiguration.Repetitions = new IntValue(repetitions);
204        parameterConfiguration.AverageQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
205        parameterConfiguration.BestQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
206        parameterConfiguration.WorstQualities = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
207        parameterConfiguration.QualityVariances = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
208        parameterConfiguration.QualityStandardDeviations = new DoubleArray(Enumerable.Repeat(0.0, problems.Count).ToArray());
209        parameterConfiguration.Runs = null;
210      }
211
212      // in OSGA there are more subscopes, so be careful which to delete
213      CurrentScope.SubScopes.RemoveAll(x => x.Variables.Count((v) => (v.Name == "RepetitionCount")) == 1);
214
215      return base.Apply();
216    }
217
218    private T1 GetResultValue<T1>(IDictionary<string, IItem> results, string resultName) {
219      return (T1)results[resultName];
220
221      //string separator = ".";
222      //string[] tokens = resultName.Split(separator.ToCharArray());
223
224      //IDictionary<string, IItem> currentResults = results;
225      //IItem currentResult = null;
226      //for (int i = 0; i < tokens.Length; i++) {
227      //  if(currentResults == null)
228      //    throw new KeyNotFoundException("Result value " + resultName + " was not found");
229      //  if (currentResults.ContainsKey(tokens[i])) {
230      //    currentResult = currentResults[tokens[i]];
231      //    currentResults = currentResult as IDictionary<string, IItem>;
232      //  } else {
233      //    throw new KeyNotFoundException("Result value " + resultName + " was not found");
234      //  }
235      //}
236      //return (T1)currentResult;
237    }
238
239    private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
240      if (ReferenceQualityAveragesParameter.ActualValue == null) {
241        // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
242        referenceQualityAverages = new double[problemsCount];
243        referenceQualityDeviations = new double[problemsCount];
244        referenceEvaluatedSolutionAverages = new double[problemsCount];
245        for (int i = 0; i < referenceQualityAverages.Length; i++) {
246          referenceQualityAverages[i] = 1;
247          referenceQualityDeviations[i] = 1;
248          referenceEvaluatedSolutionAverages[i] = 1;
249        }
250      } else {
251        referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
252        referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
253        referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
254      }
255    }
256  }
257}
Note: See TracBrowser for help on using the repository browser.