Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.MetaOptimization/HeuristicLab.Problems.MetaOptimization/3.3/Evaluators/AlgorithmRunsAnalyzer.cs @ 6018

Last change on this file since 6018 was 6018, checked in by cneumuel, 12 years ago

#1215

  • support for maximization problems
  • made base level algorithms stoppable
  • optimization for multiple goals possible (AverageQuality, AverageDeviation, AverageEvaluatedSolutions)
  • lots of fixes
File size: 10.6 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Linq;
4using HeuristicLab.Common;
5using HeuristicLab.Core;
6using HeuristicLab.Data;
7using HeuristicLab.Operators;
8using HeuristicLab.Optimization;
9using HeuristicLab.Parameters;
10using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
11
12namespace HeuristicLab.Problems.MetaOptimization {
13  [Item("AlgorithmRunsAnalyzer", "")]
14  [StorableClass]
15  public class AlgorithmRunsAnalyzer : SingleSuccessorOperator {
16
17    #region Parameter properties
18    public ILookupParameter<DoubleValue> QualityParameter {
19      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
20    }
21    public LookupParameter<IntValue> GenerationsParameter {
22      get { return (LookupParameter<IntValue>)Parameters["Generations"]; }
23    }
24    public LookupParameter<IntValue> RepetitionsParameter {
25      get { return (LookupParameter<IntValue>)Parameters[MetaOptimizationProblem.RepetitionsParameterName]; }
26    }
27    public ILookupParameter<ParameterConfigurationTree> ParameterConfigurationParameter {
28      get { return (ILookupParameter<ParameterConfigurationTree>)Parameters["ParameterConfigurationTree"]; }
29    }
30    public ILookupParameter<IItemList<IProblem>> ProblemsParameter {
31      get { return (ILookupParameter<IItemList<IProblem>>)Parameters[MetaOptimizationProblem.ProblemsParameterName]; }
32    }
33    public LookupParameter<DoubleArray> ReferenceQualityAveragesParameter {
34      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityAverages"]; }
35    }
36    public LookupParameter<DoubleArray> ReferenceQualityDeviationsParameter {
37      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceQualityDeviations"]; }
38    }
39    public LookupParameter<DoubleArray> ReferenceEvaluatedSolutionAveragesParameter {
40      get { return (LookupParameter<DoubleArray>)Parameters["ReferenceEvaluatedSolutionAverages"]; }
41    }
42    public LookupParameter<ResultCollection> ResultsParameter {
43      get { return (LookupParameter<ResultCollection>)Parameters["Results"]; }
44    }
45    public ScopeTreeLookupParameter<IAlgorithm> AlgorithmParameter {
46      get { return (ScopeTreeLookupParameter<IAlgorithm>)Parameters["Algorithm"]; }
47    }
48    public ScopeTreeLookupParameter<IntValue> ProblemIndexParameter {
49      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["ProblemIndex"]; }
50    }
51    public ScopeTreeLookupParameter<IntValue> RepetitionIndexParameter {
52      get { return (ScopeTreeLookupParameter<IntValue>)Parameters["RepetitionIndex"]; }
53    }
54    public LookupParameter<BoolValue> MaximizationParameter {
55      get { return (LookupParameter<BoolValue>)Parameters["Maximization"]; }
56    }
57    #endregion
58
59    [StorableConstructor]
60    protected AlgorithmRunsAnalyzer(bool deserializing) : base(deserializing) { }
61    public AlgorithmRunsAnalyzer() : base() {
62      Parameters.Add(new LookupParameter<IRandom>("Random", "The pseudo random number generator which should be used to initialize the new random permutation."));
63      Parameters.Add(new LookupParameter<DoubleValue>("Quality", "The evaluated quality of the ParameterVector."));
64      Parameters.Add(new LookupParameter<IntValue>("Generations", ""));
65      Parameters.Add(new LookupParameter<IntValue>(MetaOptimizationProblem.RepetitionsParameterName, "Number of evaluations on one problem."));
66      Parameters.Add(new LookupParameter<ParameterConfigurationTree>("ParameterConfigurationTree", ""));
67      Parameters.Add(new LookupParameter<IItemList<IProblem>>(MetaOptimizationProblem.ProblemsParameterName, ""));
68      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityAverages", ""));
69      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceQualityDeviations", ""));
70      Parameters.Add(new LookupParameter<DoubleArray>("ReferenceEvaluatedSolutionAverages", ""));
71      Parameters.Add(new LookupParameter<ResultCollection>("Results", ""));
72      Parameters.Add(new ScopeTreeLookupParameter<IAlgorithm>("Algorithm", "The finished algorithms containing Runs."));
73      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("ProblemIndex", "The index of the problem an algorithm was executed with."));
74      Parameters.Add(new ScopeTreeLookupParameter<IntValue>("RepetitionIndex", "The index of the repetition"));
75      Parameters.Add(new LookupParameter<BoolValue>("Maximization", "Set to false if the problem should be minimized."));
76    }
77    protected AlgorithmRunsAnalyzer(AlgorithmRunsAnalyzer original, Cloner cloner)
78      : base(original, cloner) {
79    }
80    public override IDeepCloneable Clone(Cloner cloner) {
81      return new AlgorithmRunsAnalyzer(this, cloner);
82    }
83
84    public override IOperation Apply() {
85      ParameterConfigurationTree parameterConfiguration = ParameterConfigurationParameter.ActualValue;
86      ItemArray<IAlgorithm> algorithms = AlgorithmParameter.ActualValue;
87      ItemArray<IntValue> problemIndices = ProblemIndexParameter.ActualValue;
88      ItemArray<IntValue> repetitionIndices = RepetitionIndexParameter.ActualValue;
89      IEnumerable<string> parameterNames = parameterConfiguration.GetOptimizedParameterNames();
90      IItemList<IProblem> problems = ProblemsParameter.ActualValue;
91      bool maximization = MaximizationParameter.ActualValue.Value;
92      int repetitions = RepetitionsParameter.ActualValue.Value;
93      var resultNames = new List<string> { "BestQuality", "Execution Time", "EvaluatedSolutions" };
94      int currentGeneration = GenerationsParameter.ActualValue != null ? GenerationsParameter.ActualValue.Value : 0;
95      double[] referenceQualityAverages;
96      double[] referenceQualityDeviations;
97      double[] referenceEvaluatedSolutionAverages;
98      GetReferenceValues(problems.Count, out referenceQualityAverages, out referenceQualityDeviations, out referenceEvaluatedSolutionAverages);
99
100      ResultCollection results = ResultsParameter.ActualValue;
101
102      if (algorithms.All(x => x.Runs.Count == 1)) {
103        var runs = new RunCollection();
104        var qualities = new double[problems.Count][];
105        var executionTimes = new TimeSpan[problems.Count][];
106        var evaluatedSolutions = new int[problems.Count][];
107
108        for (int i = 0; i < problems.Count; i++) {
109          qualities[i] = new double[repetitions];
110          evaluatedSolutions[i] = new int[repetitions];
111          executionTimes[i] = new TimeSpan[repetitions];
112        }
113
114        for (int i = 0; i < algorithms.Length; i++) {
115          int problemIndex = problemIndices[i].Value;
116          int repetitionIndex = repetitionIndices[i].Value;
117
118          IRun run = algorithms[i].Runs.Single();
119          MetaOptimizationUtil.ClearResults(run, resultNames);
120          MetaOptimizationUtil.ClearParameters(run, parameterNames);
121          run.Results.Add("Meta.FromCache", new BoolValue(false));
122          run.Results.Add("Meta.Generation", new IntValue(currentGeneration));
123          run.Results.Add("Meta.ProblemIndex", new IntValue(problemIndex));
124          run.Name = string.Format("{0} Problem {1} Run {2}", parameterConfiguration.ParameterInfoString, problemIndex, repetitionIndex);
125          qualities[problemIndex][repetitionIndex] = (((DoubleValue)run.Results["BestQuality"]).Value);
126          executionTimes[problemIndex][repetitionIndex] = (((TimeSpanValue)run.Results["Execution Time"]).Value);
127          evaluatedSolutions[problemIndex][repetitionIndex] = (((IntValue)run.Results["EvaluatedSolutions"]).Value);
128          runs.Add((IRun)run.Clone());
129        }
130
131        parameterConfiguration.AverageExecutionTimes = new ItemList<TimeSpanValue>(executionTimes.Select(t => new TimeSpanValue(TimeSpan.FromMilliseconds(t.Average(ts => ts.TotalMilliseconds)))));
132        parameterConfiguration.AverageEvaluatedSolutions = new DoubleArray(evaluatedSolutions.Select(x => x.Average()).ToArray());
133        parameterConfiguration.Repetitions = new IntValue(repetitions);
134        parameterConfiguration.AverageQualities = new DoubleArray(qualities.Select(q => q.Average()).ToArray());
135
136        if(maximization)
137          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
138        else
139          parameterConfiguration.BestQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
140       
141        if (maximization)
142          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Min()).ToArray());
143        else
144          parameterConfiguration.WorstQualities = new DoubleArray(qualities.Select(q => q.Max()).ToArray());
145
146        parameterConfiguration.QualityVariances = new DoubleArray(qualities.Select(q => q.Variance()).ToArray());
147        parameterConfiguration.QualityStandardDeviations = new DoubleArray(qualities.Select(q => q.StandardDeviation()).ToArray());
148        parameterConfiguration.Runs = runs;
149
150        this.QualityParameter.ActualValue = new DoubleValue(MetaOptimizationUtil.Normalize(parameterConfiguration, referenceQualityAverages, referenceQualityDeviations, referenceEvaluatedSolutionAverages, 1, 1, 1, maximization));
151      } else {
152        // something terrible happened -> most probably due to invalid parameters.
153        // penalty with worst quality from latest generation!
154
155        double penaltyValue = results.ContainsKey("CurrentWorstQuality") ? ((DoubleValue)results["CurrentWorstQuality"]).Value : referenceQualityAverages.Max(); // todo: respect min/max
156        this.QualityParameter.ActualValue = new DoubleValue(penaltyValue);
157      }
158
159      return base.Apply();
160    }
161
162    private void GetReferenceValues(int problemsCount, out double[] referenceQualityAverages, out double[] referenceQualityDeviations, out double[] referenceEvaluatedSolutionAverages) {
163      if (ReferenceQualityAveragesParameter.ActualValue == null) {
164        // this is generation zero. no reference qualities for normalization have been calculated yet. in this special case the ReferenceQualityAnalyzer will do the normalization
165        referenceQualityAverages = new double[problemsCount];
166        referenceQualityDeviations = new double[problemsCount];
167        referenceEvaluatedSolutionAverages = new double[problemsCount];
168        for (int i = 0; i < referenceQualityAverages.Length; i++) {
169          referenceQualityAverages[i] = 1;
170          referenceQualityDeviations[i] = 1;
171          referenceEvaluatedSolutionAverages[i] = 1;
172        }
173      } else {
174        referenceQualityAverages = ReferenceQualityAveragesParameter.ActualValue.ToArray();
175        referenceQualityDeviations = ReferenceQualityDeviationsParameter.ActualValue.ToArray();
176        referenceEvaluatedSolutionAverages = ReferenceEvaluatedSolutionAveragesParameter.ActualValue.ToArray();
177      }
178    }
179  }
180}
Note: See TracBrowser for help on using the repository browser.