1 | namespace HeuristicLab.Problems.ProgramSynthesis.Push.Problem.BenchmarkSuite {
|
---|
2 | using System;
|
---|
3 | using System.Collections.Generic;
|
---|
4 | using Common;
|
---|
5 | using Core;
|
---|
6 | using Expressions;
|
---|
7 |
|
---|
8 | using HeuristicLab.BenchmarkSuite;
|
---|
9 | using HeuristicLab.BenchmarkSuite.Problems;
|
---|
10 | using HeuristicLab.Parameters;
|
---|
11 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
12 | using HeuristicLab.Problems.ProgramSynthesis.Push.Configuration;
|
---|
13 |
|
---|
14 | using Interpreter;
|
---|
15 | using Problem;
|
---|
16 | using Stack;
|
---|
17 |
|
---|
18 | [StorableClass]
|
---|
19 | public class PushBenchmarkSuiteEvaluator : ParameterizedNamedItem, IPushEvaluator {
|
---|
20 |
|
---|
21 | private const string DataBoundsParameterName = "DataBounds";
|
---|
22 | private const string DataParameterName = "Data";
|
---|
23 | private const string DataParameterDescription = "Program Synthesis";
|
---|
24 |
|
---|
25 | public PushBenchmarkSuiteEvaluator() {
|
---|
26 | Parameters.Add(new FixedValueParameter<DataBounds>(DataBoundsParameterName));
|
---|
27 |
|
---|
28 | Parameters.Add(new ValueParameter<ProblemData>(
|
---|
29 | DataParameterName,
|
---|
30 | DataParameterDescription));
|
---|
31 | }
|
---|
32 |
|
---|
33 | public PushBenchmarkSuiteEvaluator(ProblemData data) : this() {
|
---|
34 | LoadData(data);
|
---|
35 | }
|
---|
36 |
|
---|
37 | [StorableConstructor]
|
---|
38 | public PushBenchmarkSuiteEvaluator(bool deserializing) { }
|
---|
39 |
|
---|
40 | public PushBenchmarkSuiteEvaluator(PushBenchmarkSuiteEvaluator origin, Cloner cloner) : base(origin, cloner) {
|
---|
41 | Data = cloner.Clone(origin.Data);
|
---|
42 | }
|
---|
43 |
|
---|
44 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
45 | return new PushBenchmarkSuiteEvaluator(this, cloner);
|
---|
46 | }
|
---|
47 |
|
---|
48 | public IValueParameter<ProblemData> DataParameter
|
---|
49 | {
|
---|
50 | get { return (IValueParameter<ProblemData>)Parameters[DataParameterName]; }
|
---|
51 | }
|
---|
52 |
|
---|
53 | public ProblemData Data
|
---|
54 | {
|
---|
55 | get { return DataParameter.Value; }
|
---|
56 | set { DataParameter.Value = value; }
|
---|
57 | }
|
---|
58 |
|
---|
59 | public IValueParameter<DataBounds> DataBoundsParameter
|
---|
60 | {
|
---|
61 | get { return (IValueParameter<DataBounds>)Parameters[DataBoundsParameterName]; }
|
---|
62 | }
|
---|
63 |
|
---|
64 | public DataBounds DataBounds
|
---|
65 | {
|
---|
66 | get { return DataBoundsParameter.Value; }
|
---|
67 | set { DataBoundsParameter.Value = value; }
|
---|
68 | }
|
---|
69 |
|
---|
70 | public void LoadData(ProblemData data) {
|
---|
71 | this.Data = data;
|
---|
72 |
|
---|
73 | DataBounds.TrainingRange.Start = 0;
|
---|
74 | DataBounds.TrainingRange.End = Data.TrainingCount;
|
---|
75 | DataBounds.TestRange.Start = Data.TrainingCount;
|
---|
76 | DataBounds.TestRange.End = Data.TrainingCount + Data.TestCount;
|
---|
77 | }
|
---|
78 |
|
---|
79 | public EvaluationResult Evaluate(IPushInterpreter interpreter, PushProgram program, IRandom random, int start, int end) {
|
---|
80 | var length = end - start;
|
---|
81 | if (length <= 0) return null;
|
---|
82 |
|
---|
83 | var evaluationResult = new EvaluationResult(length);
|
---|
84 |
|
---|
85 | for (var i = start; i < end; i++) {
|
---|
86 | var result = Evaluate(interpreter, program, i);
|
---|
87 | evaluationResult.ExampleQualities[i - start] = result;
|
---|
88 | evaluationResult.TotalQuality += result;
|
---|
89 | interpreter.Reset();
|
---|
90 | }
|
---|
91 |
|
---|
92 | evaluationResult.TotalQuality /= length;
|
---|
93 |
|
---|
94 | return evaluationResult;
|
---|
95 | }
|
---|
96 |
|
---|
97 | public EvaluationResult EvaluateTest(IPushInterpreter interpreter, PushProgram program, IRandom random) {
|
---|
98 | return Evaluate(interpreter, program, random, DataBounds.TestRange.Start, DataBounds.TestRange.End);
|
---|
99 | }
|
---|
100 |
|
---|
101 | public EvaluationResult EvaluateTest(IReadOnlyPushConfiguration config, PushProgram program, IRandom random) {
|
---|
102 | var interpreter = new PushInterpreter(config, random);
|
---|
103 | return EvaluateTest(interpreter, program, random);
|
---|
104 | }
|
---|
105 |
|
---|
106 | public EvaluationResult EvaluateTest(PushInterpreterPool pool, PushProgram program, IRandom random) {
|
---|
107 | using (var interpreter = pool.Create(random)) {
|
---|
108 | return EvaluateTest(interpreter, program, random);
|
---|
109 | }
|
---|
110 | }
|
---|
111 |
|
---|
112 | public EvaluationResult EvaluateTraining(IPushInterpreter interpreter, PushProgram program, IRandom random) {
|
---|
113 | return Evaluate(interpreter, program, random, DataBounds.TrainingRange.Start, DataBounds.TrainingRange.End);
|
---|
114 | }
|
---|
115 |
|
---|
116 | public EvaluationResult EvaluateTraining(IReadOnlyPushConfiguration config, PushProgram program, IRandom random) {
|
---|
117 | var interpreter = new PushInterpreter(config, random);
|
---|
118 | return EvaluateTraining(interpreter, program, random);
|
---|
119 | }
|
---|
120 |
|
---|
121 | public EvaluationResult EvaluateTraining(PushInterpreterPool pool, PushProgram program, IRandom random) {
|
---|
122 | using (var interpreter = pool.Create(random)) {
|
---|
123 | return EvaluateTraining(interpreter, program, random);
|
---|
124 | }
|
---|
125 | }
|
---|
126 |
|
---|
127 | public double Evaluate(IPushInterpreter interpreter, PushProgram program, int exampleIndex) {
|
---|
128 | var example = Data.Examples[exampleIndex];
|
---|
129 |
|
---|
130 | interpreter.BooleanStack.Push(example.InputBoolean);
|
---|
131 | interpreter.IntegerStack.Push(example.InputInteger);
|
---|
132 | interpreter.FloatStack.Push(example.InputFloat);
|
---|
133 |
|
---|
134 | interpreter.Run(program);
|
---|
135 |
|
---|
136 | var result = GetDiff(example.OutputInteger, interpreter.IntegerStack, Data.WorstResult, IntegerDiffer)
|
---|
137 | + GetDiff(example.OutputFloat, interpreter.FloatStack, Data.WorstResult, FloatDiffer)
|
---|
138 | + GetDiff(example.OutputBoolean, interpreter.BooleanStack, Data.WorstResult, BooleanDiffer)
|
---|
139 | + GetDiff(example.OutputString, interpreter.StringStack, Data.WorstResult, StringDiffer)
|
---|
140 | + GetDiff(example.OutputChar, interpreter.CharStack, Data.WorstResult, CharDiffer)
|
---|
141 | + GetPrintDiffer(example.OutputPrint, interpreter.PrintStack)
|
---|
142 | + GetVectorDiff(example.OutputIntegerVector, interpreter.IntegerVectorStack, Data.WorstResult, (a, b) => VectorDiffer(a, b, IntegerDiffer))
|
---|
143 | + GetVectorDiff(example.OutputFloatVector, interpreter.FloatVectorStack, Data.WorstResult, (a, b) => VectorDiffer(a, b, FloatDiffer))
|
---|
144 | + GetVectorDiff(example.OutputStringVector, interpreter.StringVectorStack, Data.WorstResult, (a, b) => VectorDiffer(a, b, StringDiffer));
|
---|
145 |
|
---|
146 | return result;
|
---|
147 | }
|
---|
148 |
|
---|
149 | private static double FloatDiffer(double a, double b) {
|
---|
150 | var result = a - b;
|
---|
151 |
|
---|
152 | if (result.IsAlmost(double.MinValue) || double.IsPositiveInfinity(result) || double.IsNaN(result))
|
---|
153 | return double.MaxValue;
|
---|
154 |
|
---|
155 | if (result.IsAlmost(double.MaxValue) || double.IsNegativeInfinity(result))
|
---|
156 | return double.MinValue;
|
---|
157 |
|
---|
158 | return Math.Abs(result);
|
---|
159 | }
|
---|
160 |
|
---|
161 | private static double IntegerDiffer(long a, long b) {
|
---|
162 | var result = a - b;
|
---|
163 |
|
---|
164 | return result == long.MinValue ? long.MaxValue : Math.Abs(result);
|
---|
165 | }
|
---|
166 |
|
---|
167 | private static double BooleanDiffer(bool a, bool b) {
|
---|
168 | return a && b ? 0 : a || b ? 1 : 2;
|
---|
169 | }
|
---|
170 |
|
---|
171 | private static double StringDiffer(string a, string b) {
|
---|
172 | return LevenshteinDistance(a, b);
|
---|
173 | }
|
---|
174 |
|
---|
175 | private static double CharDiffer(char a, char b) {
|
---|
176 | return IntegerDiffer(a, b);
|
---|
177 | }
|
---|
178 |
|
---|
179 | private static double VectorDiffer<T>(IReadOnlyList<T> estimated, IReadOnlyList<T> outcome, Func<T, T, double> differ) {
|
---|
180 | var length = Math.Min(estimated.Count, outcome.Count);
|
---|
181 | var result = 0d;
|
---|
182 |
|
---|
183 | for (var i = 0; i < length; i++) {
|
---|
184 | result += differ(estimated[i], outcome[i]);
|
---|
185 | }
|
---|
186 |
|
---|
187 | if (estimated.Count > outcome.Count) {
|
---|
188 | var remainingLength = estimated.Count - length;
|
---|
189 | for (var i = 0; i < remainingLength; i++)
|
---|
190 | result += differ(estimated[i], default(T));
|
---|
191 | }
|
---|
192 |
|
---|
193 | return result;
|
---|
194 | }
|
---|
195 |
|
---|
196 | private static double GetPrintDiffer(string estimated, IPushStack<string> printStack) {
|
---|
197 | var printResult = string.Join(Environment.NewLine, printStack);
|
---|
198 |
|
---|
199 | return LevenshteinDistance(estimated, printResult);
|
---|
200 | }
|
---|
201 |
|
---|
202 | private static double GetVectorDiff<T>(IReadOnlyList<IReadOnlyList<T>> estimated, IPushStack<List<T>> resultStack, double worstResult, Func<IReadOnlyList<T>, IReadOnlyList<T>, double> differ)
|
---|
203 | where T : IComparable {
|
---|
204 | if (estimated.Count == 0) return 0d;
|
---|
205 |
|
---|
206 | var diff = 0d;
|
---|
207 | var comparableLength = 0;
|
---|
208 |
|
---|
209 | if (!resultStack.IsEmpty) {
|
---|
210 | var count = Math.Min(estimated.Count, resultStack.Count);
|
---|
211 | var result = resultStack.Peek(count);
|
---|
212 | comparableLength = Math.Min(estimated.Count, result.Length);
|
---|
213 |
|
---|
214 | for (var i = 0; i < comparableLength; i++) {
|
---|
215 | diff += Math.Min(differ(estimated[i], result[i]), worstResult);
|
---|
216 | }
|
---|
217 | }
|
---|
218 |
|
---|
219 | var emptyTArray = new T[0];
|
---|
220 | for (var i = comparableLength; i < estimated.Count - comparableLength; i++) {
|
---|
221 | diff += differ(estimated[i], emptyTArray);
|
---|
222 | }
|
---|
223 |
|
---|
224 | return diff;
|
---|
225 | }
|
---|
226 |
|
---|
227 | private static double GetDiff<T>(IReadOnlyList<T> estimated, IPushStack<T> resultStack, double worstResult, Func<T, T, double> differ)
|
---|
228 | where T : IComparable {
|
---|
229 | if (estimated.Count == 0) return 0d;
|
---|
230 |
|
---|
231 | var diff = 0d;
|
---|
232 | var comparableLength = 0;
|
---|
233 |
|
---|
234 | if (!resultStack.IsEmpty) {
|
---|
235 | var count = Math.Min(estimated.Count, resultStack.Count);
|
---|
236 | var result = resultStack.Peek(count);
|
---|
237 | comparableLength = Math.Min(estimated.Count, result.Length);
|
---|
238 |
|
---|
239 | for (var i = 0; i < comparableLength; i++) {
|
---|
240 | diff += Math.Min(differ(estimated[i], result[i]), worstResult);
|
---|
241 | }
|
---|
242 | }
|
---|
243 |
|
---|
244 | for (var i = comparableLength; i < estimated.Count - comparableLength; i++) {
|
---|
245 | diff += differ(estimated[i], default(T));
|
---|
246 | }
|
---|
247 |
|
---|
248 | return diff;
|
---|
249 | }
|
---|
250 |
|
---|
251 | /// <summary>
|
---|
252 | /// https://www.dotnetperls.com/levenshtein
|
---|
253 | /// </summary>
|
---|
254 | private static int LevenshteinDistance(string source, string target) {
|
---|
255 | if (source == null && target == null) return 0;
|
---|
256 | if (source == null) return target.Length;
|
---|
257 | if (target == null) return source.Length;
|
---|
258 |
|
---|
259 | int n = source.Length;
|
---|
260 | int m = target.Length;
|
---|
261 | int[,] d = new int[n + 1, m + 1];
|
---|
262 |
|
---|
263 | // Step 1
|
---|
264 | if (n == 0) {
|
---|
265 | return m;
|
---|
266 | }
|
---|
267 |
|
---|
268 | if (m == 0) {
|
---|
269 | return n;
|
---|
270 | }
|
---|
271 |
|
---|
272 | // Step 2
|
---|
273 | for (int i = 0; i <= n; d[i, 0] = i++) {
|
---|
274 | }
|
---|
275 |
|
---|
276 | for (int j = 0; j <= m; d[0, j] = j++) {
|
---|
277 | }
|
---|
278 |
|
---|
279 | // Step 3
|
---|
280 | for (int i = 1; i <= n; i++) {
|
---|
281 | //Step 4
|
---|
282 | for (int j = 1; j <= m; j++) {
|
---|
283 | // Step 5
|
---|
284 | int cost = (target[j - 1] == source[i - 1]) ? 0 : 1;
|
---|
285 |
|
---|
286 | // Step 6
|
---|
287 | d[i, j] = Math.Min(
|
---|
288 | Math.Min(d[i - 1, j] + 1, d[i, j - 1] + 1),
|
---|
289 | d[i - 1, j - 1] + cost);
|
---|
290 | }
|
---|
291 | }
|
---|
292 | // Step 7
|
---|
293 | return d[n, m];
|
---|
294 | }
|
---|
295 | }
|
---|
296 | }
|
---|