Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs @ 5653

Last change on this file since 5653 was 5653, checked in by cneumuel, 12 years ago

#1215

  • evaluation operator returns operatorgraph which creates a scope and an operation for each algorithm execution (each repetition and problem)
  • split ValueConfiguration into ParameterizedValueConfiguration and RangeValueConfiguration
File size: 8.3 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Linq;
4using System.Text;
5using HeuristicLab.Operators;
6using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
7using HeuristicLab.Core;
8using HeuristicLab.Common;
9using HeuristicLab.Parameters;
10using HeuristicLab.Data;
11using HeuristicLab.Optimization;
12
13namespace HeuristicLab.Problems.MetaOptimization {
14  [Item("AlgorithmRunsAnalyzer", "")]
15  [StorableClass]
16  public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
17
18    #region Parameter properties
19    public ILookupParameter<DoubleValue> QualityParameter {
20      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
21    }
22    public LookupParameter<IntValue> GenerationsParameter {
23      get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
24    }
25    public LookupParameter<IntValue> RepetitionsParameter {
26      get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
27    }
28    public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
29      get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
30    }
31    public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
32      get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
33    }
34    public LookupParameter<DoubleArray> ProblemQualityReferencesParameter {
35      get { return (LookupParameter<DoubleArray>)Parameters["ProblemQualityReferences"]; }
36    }
37    public LookupParameter<ResultCollection> ResultsParameter {
38      get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
39    }
40    public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
41      get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
42    }
43    public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
44      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
45    }
46    public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
47      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
48    }
49    #endregion
50
51    [StorableConstructor]
52    protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
53    public AlgorithmRunsAnalyzer() : base() {
54      Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
55      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
56      Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
57      Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
58      Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
59      Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
60      Parameters.Add(new LookupParameter<DoubleArray>("ProblemQualityReferences", ""));
61      Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
62      Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
63      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
64      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
65    }
66    protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
67      : base(original, cloner) {
68    }
69    public override IDeepCloneable Clone(Cloner cloner) {
70      return new AlgorithmRunsAnalyzer(this, cloner);
71    }
72
73    public override IOperation Apply() {
74      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
75      ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
76      ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
77      ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
78      IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
79      IItemList<IProblem> problems = ProblemsParameter.ActualValue;
80      int repetitions = RepetitionsParameter.ActualValue.Value;
81      var resultNames = new List<string> { "BestQuality", "Execution Time" };
82      int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
83      double[] referenceQualities = GetReferenceQualities(problems.Count);
84      ResultCollection results = ResultsParameter.ActualValue;
85
86      if (algorithms.All(x => x.Runs.Count == 1)) {
87        var runs = new RunCollection();
88        var qualities = new double[problems.Count][];
89        var executionTimes = new TimeSpan[problems.Count][];
90
91        for (int i = 0; i < problems.Count; i++) {
92          qualities[i] = new double[repetitions];
93          executionTimes[i] = new TimeSpan[repetitions];
94        }
95
96        for (int i = 0; i < algorithms.Length; i++) {
97          int problemIndex = problemIndices[i].Value;
98          int repetitionIndex = repetitionIndices[i].Value;
99
100          IRun run = algorithms[i].Runs.First();
101          MetaOptimizationUtil.ClearResults(run, resultNames);
102          MetaOptimizationUtil.ClearParameters(run, parameterNames);
103          run.Results.Add("Meta.FromCache", new BoolValue(false));
104          run.Results.Add("Meta.Generation", new IntValue(currentGeneration));
105          run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
106          run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
107          qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
108          executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
109          runs.Add((IRun)run.Clone());
110        }
111
112        parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
113        parameterConfiguration.Repetitions = new IntValue(repetitions);
114        parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray()); // todo: respect Maximization:true/false
115        parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
116        parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray()); // todo: respect Maximization:true/false
117        parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
118        parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
119        parameterConfiguration.Runs = runs;
120
121        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.NormalizeQualities(parameterConfiguration, referenceQualities));
122      } else {
123        // something terrible happened -> most probably due to invalid parameters.
124        // penalty with worst quality from latest generation!
125
126        double penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualities.Max(); // todo: respect min/max
127        this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
128      }
129
130      return base.Apply();
131    }
132
133    private double[] GetReferenceQualities(int problemsCount) {
134      double[] referenceQualities;
135      if (ProblemQualityReferencesParameter.ActualValue == null) {
136        // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
137        referenceQualities = new double[problemsCount];
138        for (int i = 0; i < referenceQualities.Length; i++) {
139          referenceQualities[i] = 1;
140        }
141      } else {
142        referenceQualities = ProblemQualityReferencesParameter.ActualValue.ToArray();
143      }
144      return referenceQualities;
145    }
146  }
147}
Note: See TracBrowser for help on using the repository browser.