Free cookie consent management tool by TermsFeed Policy Generator

source: branches/3133_ProblemModifiers/HeuristicLab.Problems.Modifiers/Modelling/AnalysisRunningPredictionQualityProblemModifier.cs @ 18029

Last change on this file since 18029 was 18029, checked in by bwerth, 3 years ago

#3133 added implementation of problem modifiers

File size: 8.7 KB
Line 
1#region License Information
2
3/* HeuristicLab
4 * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
5 *
6 * This file is part of HeuristicLab.
7 *
8 * HeuristicLab is free software: you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation, either version 3 of the License, or
11 * (at your option) any later version.
12 *
13 * HeuristicLab is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
20 */
21
22#endregion License Information
23
24using System;
25using System.Collections.Generic;
26using System.Linq;
27using HEAL.Attic;
28using HeuristicLab.Algorithms.DataAnalysis;
29using HeuristicLab.Common;
30using HeuristicLab.Core;
31using HeuristicLab.Data;
32using HeuristicLab.Encodings.IntegerVectorEncoding;
33using HeuristicLab.Encodings.RealVectorEncoding;
34using HeuristicLab.Optimization;
35using HeuristicLab.Parameters;
36using HeuristicLab.Problems.DataAnalysis;
37
38namespace HeuristicLab.Problems.Modifiers {
39
40  [StorableType("A0E33EDB-04F6-48B6-BB10-7E3753841AEA")]
41  [Item("AnalysisRunningPredictionQualityProblemModifier", " A problem modifier that provides extended Analysis by creating running models (models trained on the evaluations of previous iterations) and analyzing their performance over time")]
42  public class AnalysisRunningPredictionQualityProblemModifier : ProblemModifier {
43
44    #region Properties
45    [Storable]
46    private ModifiableDataset data;
47
48    [Storable]
49    private Dictionary<string, List<double[]>> evaluationsLookUp;
50
51    [Storable]
52    private List<Tuple<double[], double[]>> evaluatedThisIteration;
53
54    [Storable]
55    private List<Tuple<double[], double[]>> lastPopulation;
56
57    [Storable]
58    private int iteration;
59
60    [Storable]
61    private int trainingLength;
62
63    public const string ModelBuilderParameterName = "ModelBuilder";
64
65    public IValueParameter<IAlgorithm> ModelBuilderParameter => (IValueParameter<IAlgorithm>)Parameters[ModelBuilderParameterName];
66
67    public IAlgorithm ModelBuilder => ModelBuilderParameter.Value;
68
69    #endregion
70    [StorableConstructor]
71    protected AnalysisRunningPredictionQualityProblemModifier(StorableConstructorFlag _) : base(_) { }
72
73    protected AnalysisRunningPredictionQualityProblemModifier(AnalysisRunningPredictionQualityProblemModifier original, Cloner cloner) : base(original, cloner) {
74      data = cloner.Clone(original?.data);
75      evaluationsLookUp = original?.evaluationsLookUp.ToDictionary(e => e.Key, e => e.Value.Select(o => o.ToArray()).ToList());
76      iteration = original?.iteration ?? 0;
77      trainingLength = original?.trainingLength ?? 0;
78      evaluatedThisIteration = original?.evaluatedThisIteration.Select(x => Tuple.Create(x.Item1.ToArray(), x.Item2.ToArray())).ToList();
79      lastPopulation = original?.lastPopulation.Select(x => Tuple.Create(x.Item1.ToArray(), x.Item2.ToArray())).ToList();
80      Parameters.Add(new ValueParameter<IAlgorithm>(ModelBuilderParameterName, "The model builder", new GaussianProcessRegression()));
81    }
82
83    protected AnalysisRunningPredictionQualityProblemModifier() {
84      evaluationsLookUp = new Dictionary<string, List<double[]>>();
85    }
86
87    public override void Initialize() {
88      data = new ModifiableDataset();
89      if (evaluationsLookUp == null) evaluationsLookUp = new Dictionary<string, List<double[]>>();
90      evaluationsLookUp.Clear();
91      iteration = 0;
92      trainingLength = 0;
93      evaluatedThisIteration = new List<Tuple<double[], double[]>>();
94      lastPopulation = new List<Tuple<double[], double[]>>();
95    }
96
97    public override void ModifiedAnalyze(Individual[] individuals, double[][] qualities, ResultCollection results, IRandom random) {
98      var models = new ResultCollection(qualities.First().Length);
99      for (var i = 0; i < qualities.First().Length; i++) {
100        var pd = new RegressionProblemData(data, data.VariableNames.Where(v => v.Contains("X")), TargetVariableName(i));
101        pd.TrainingPartition.Start = 0;
102        pd.TrainingPartition.End = pd.TestPartition.Start = trainingLength;
103        pd.TestPartition.End = data.Rows;
104        models.AddOrUpdateResult(TargetVariableName(i), BuildRunningModel(pd, random));
105      }
106      results.AddOrUpdateResult("Running Models", models);
107      trainingLength = data.Rows;
108      lastPopulation = individuals.Zip(qualities, (i, q) => Tuple.Create(ExtractInputs(i), q)).ToList();
109      evaluatedThisIteration.Clear();
110      iteration++;
111      base.ModifiedAnalyze(individuals, qualities, results, random);
112    }
113
114    public override double[] ModifiedEvaluate(Individual individual, IRandom random) {
115      var q = base.ModifiedEvaluate(individual, random);
116      lock (data) {
117        ExtendDatasetWithoutDuplicates(new[] { individual }, new[] { q });
118        evaluatedThisIteration.Add(Tuple.Create(ExtractInputs(individual), q.ToArray()));
119      }
120      return q;
121    }
122
123    private IRegressionSolution BuildRunningModel(RegressionProblemData pd, IRandom random) {
124      if (pd.TrainingPartition.Size <= 0) return null;
125      try {
126        ModelBuilder.Problem = new RegressionProblem() { ProblemData = pd };
127        if (ModelBuilder.Parameters.ContainsKey("Seed") && (ModelBuilder.Parameters["Seed"] is IValueParameter<IntValue> seedParam)) seedParam.Value.Value = random.Next();
128        if (ModelBuilder.Parameters.ContainsKey("SetSeedRandomly") && (ModelBuilder.Parameters["SetSeedRandomly"] is IValueParameter<BoolValue> setSeedParam)) setSeedParam.Value.Value = false;
129        ModelBuilder.Start();
130        var res = ModelBuilder.Results.Select(x => x.Value).OfType<IRegressionSolution>().Single();
131        ModelBuilder.Prepare();
132        ModelBuilder.Runs.Clear();
133        return res;
134      } catch (Exception) {
135        return null;
136      }
137    }
138
139
140    #region DataHandling
141    private void ExtendDatasetWithoutDuplicates(IReadOnlyList<Individual> individuals, IReadOnlyList<double[]> qualities) {
142      if (data.Rows == 0) {
143        for (var i = 0; i < ExtractInputs(individuals[0]).Length; i++) {
144          var v = InputVariableName(i);
145          if (!data.DoubleVariables.Contains(v))
146            data.AddVariable(v, new List<double>());
147        }
148        for (var i = 0; i < qualities[0].Length; i++) {
149          var v = TargetVariableName(i);
150          if (!data.DoubleVariables.Contains(v))
151            data.AddVariable(v, new List<double>());
152        }
153      }
154      for (var i = 0; i < individuals.Count; i++) {
155        var ins = ExtractInputs(individuals[i]);
156        var id = ToIdentifier(ins);
157        var outs = qualities[i];
158        if (outs.Any(x => double.IsNaN(x) || double.IsInfinity(x) || double.MaxValue / 100 < x || double.MinValue / 100 > x)) continue;
159        if (evaluationsLookUp.ContainsKey(id) && evaluationsLookUp[id].Any(o => Equals(o, outs))) continue;
160        if (ins.Length + outs.Length != data.DoubleVariables.Count()) throw new ArgumentException("length of individuals and outputs does not match existing data");
161        data.AddRow(ins.Concat(qualities[i]).Select(x => (object)x));
162        if (!evaluationsLookUp.ContainsKey(id)) evaluationsLookUp.Add(id, new List<double[]>() { outs });
163        else { evaluationsLookUp[id].Add(outs); }
164      }
165    }
166
167    private static double[] ExtractInputs(Individual individual) {
168      if (!(individual is SingleEncodingIndividual si)) throw new ArgumentException("Multi encodings are not supported with this problem modifier");
169      switch (si[si.Name]) {
170        case RealVector rv:
171          return rv.CloneAsArray();
172        case IntegerVector iv:
173          return iv.Select(i => (double)i).ToArray();
174        default:
175          throw new ArgumentException("Only Integer and Real Vector Individuals can be transformed to input values");
176      }
177    }
178    #endregion DataHandling
179
180    #region Naming
181    public static string ToIdentifier(IEnumerable<double> inputs) {
182      return string.Join(";", inputs);
183    }
184
185    public static string ToIdentifier(Individual i) {
186      return string.Join(";", ExtractInputs(i));
187    }
188
189    public static string TargetVariableName(int targetNumber) {
190      return "Y" + targetNumber;
191    }
192
193    public static string InputVariableName(int inputNumber) {
194      return "X" + inputNumber;
195    }
196    #endregion Naming
197    public override IDeepCloneable Clone(Cloner cloner) {
198      return new AnalysisRunningPredictionQualityProblemModifier(this, cloner);
199    }
200  }
201}
Note: See TracBrowser for help on using the repository browser.