1 | namespace HeuristicLab.Tests.Benchmark.Problem {
|
---|
2 | using System;
|
---|
3 | using System.Collections.Generic;
|
---|
4 | using System.Linq;
|
---|
5 | using System.Threading.Tasks;
|
---|
6 |
|
---|
7 | using HeuristicLab.BenchmarkSuite;
|
---|
8 | using HeuristicLab.BenchmarkSuite.Problems;
|
---|
9 | using HeuristicLab.Problems.Instances;
|
---|
10 | using HeuristicLab.Problems.ProgramSynthesis.Push.Configuration;
|
---|
11 | using HeuristicLab.Problems.ProgramSynthesis.Push.Data.Pool;
|
---|
12 | using HeuristicLab.Problems.ProgramSynthesis.Push.Expressions;
|
---|
13 | using HeuristicLab.Problems.ProgramSynthesis.Push.Generators;
|
---|
14 | using HeuristicLab.Problems.ProgramSynthesis.Push.Interpreter;
|
---|
15 | using HeuristicLab.Problems.ProgramSynthesis.Push.Stack;
|
---|
16 | using HeuristicLab.Random;
|
---|
17 |
|
---|
18 | using Microsoft.VisualStudio.TestTools.UnitTesting;
|
---|
19 |
|
---|
20 | [TestClass]
|
---|
21 | public class ProblemTests {
|
---|
22 | [TestMethod]
|
---|
23 | [TestProperty("Time", "Medium")]
|
---|
24 | [TestCategory("ProblemTest")]
|
---|
25 | public void CountOdds() {
|
---|
26 | RandomWalk(new CountOdds());
|
---|
27 | }
|
---|
28 |
|
---|
29 | [TestMethod]
|
---|
30 | [TestProperty("Time", "Medium")]
|
---|
31 | [TestCategory("ProblemTest")]
|
---|
32 | public void Checksum() {
|
---|
33 | RandomWalk(new Checksum());
|
---|
34 | }
|
---|
35 |
|
---|
36 | //[TestMethod]
|
---|
37 | //[TestProperty("Time", "Medium")]
|
---|
38 | //[TestCategory("ProblemTest")]
|
---|
39 | //public void CollatzNumbers() {
|
---|
40 | // RandomWalk(new CollatzNumbers());
|
---|
41 | //}
|
---|
42 |
|
---|
43 | //[TestMethod]
|
---|
44 | //[TestProperty("Time", "Medium")]
|
---|
45 | //[TestCategory("ProblemTest")]
|
---|
46 | //public void CompareStringLengths() {
|
---|
47 | // RandomWalk(new CompareStringLengths());
|
---|
48 | //}
|
---|
49 |
|
---|
50 | //[TestMethod]
|
---|
51 | //[TestProperty("Time", "Medium")]
|
---|
52 | //[TestCategory("ProblemTest")]
|
---|
53 | //public void Digits() {
|
---|
54 | // RandomWalk(new Digits());
|
---|
55 | //}
|
---|
56 |
|
---|
57 | //[TestMethod]
|
---|
58 | //[TestProperty("Time", "Medium")]
|
---|
59 | //[TestCategory("ProblemTest")]
|
---|
60 | //public void DoubleLetters() {
|
---|
61 | // RandomWalk(new DoubleLetters());
|
---|
62 | //}
|
---|
63 |
|
---|
64 | //[TestMethod]
|
---|
65 | //[TestProperty("Time", "Medium")]
|
---|
66 | //[TestCategory("ProblemTest")]
|
---|
67 | //public void EvenSquares() {
|
---|
68 | // RandomWalk(new EvenSquares());
|
---|
69 | //}
|
---|
70 |
|
---|
71 | //[TestMethod]
|
---|
72 | //[TestProperty("Time", "Medium")]
|
---|
73 | //[TestCategory("ProblemTest")]
|
---|
74 | //public void ForLoopIndex() {
|
---|
75 | // RandomWalk(new ForLoopIndex());
|
---|
76 | //}
|
---|
77 |
|
---|
78 | //[TestMethod]
|
---|
79 | //[TestProperty("Time", "Medium")]
|
---|
80 | //[TestCategory("ProblemTest")]
|
---|
81 | //public void Grades() {
|
---|
82 | // RandomWalk(new Grades());
|
---|
83 | //}
|
---|
84 |
|
---|
85 | //[TestMethod]
|
---|
86 | //[TestProperty("Time", "Medium")]
|
---|
87 | //[TestCategory("ProblemTest")]
|
---|
88 | //public void LastIndexOfZero() {
|
---|
89 | // RandomWalk(new LastIndexOfZero());
|
---|
90 | //}
|
---|
91 |
|
---|
92 | //[TestMethod]
|
---|
93 | //[TestProperty("Time", "Medium")]
|
---|
94 | //[TestCategory("ProblemTest")]
|
---|
95 | //public void Median() {
|
---|
96 | // RandomWalk(new Median());
|
---|
97 | //}
|
---|
98 |
|
---|
99 | //[TestMethod]
|
---|
100 | //[TestProperty("Time", "Medium")]
|
---|
101 | //[TestCategory("ProblemTest")]
|
---|
102 | //public void MirrorImage() {
|
---|
103 | // RandomWalk(new MirrorImage());
|
---|
104 | //}
|
---|
105 |
|
---|
106 | //[TestMethod]
|
---|
107 | //[TestProperty("Time", "Medium")]
|
---|
108 | //[TestCategory("ProblemTest")]
|
---|
109 | //public void NegativeToZero() {
|
---|
110 | // RandomWalk(new NegativeToZero());
|
---|
111 | //}
|
---|
112 |
|
---|
113 | //[TestMethod]
|
---|
114 | //[TestProperty("Time", "Medium")]
|
---|
115 | //[TestCategory("ProblemTest")]
|
---|
116 | //public void NumberIo() {
|
---|
117 | // RandomWalk(new NumberIo());
|
---|
118 | //}
|
---|
119 |
|
---|
120 | //[TestMethod]
|
---|
121 | //[TestProperty("Time", "Medium")]
|
---|
122 | //[TestCategory("ProblemTest")]
|
---|
123 | //public void PigLatin() {
|
---|
124 | // RandomWalk(new PigLatin());
|
---|
125 | //}
|
---|
126 |
|
---|
127 | //[TestMethod]
|
---|
128 | //[TestProperty("Time", "Medium")]
|
---|
129 | //[TestCategory("ProblemTest")]
|
---|
130 | //public void ReplaceSpaceWithNewLine() {
|
---|
131 | // RandomWalk(new ReplaceSpaceWithNewline());
|
---|
132 | //}
|
---|
133 |
|
---|
134 | private static void RandomWalk(IDataDescriptor descriptor) {
|
---|
135 | var maxProgramSizeLimit = 1024;
|
---|
136 | var iterations = 100;
|
---|
137 | var best = double.MaxValue;
|
---|
138 | var globalExecCounter = 0;
|
---|
139 | var lockObj = new object();
|
---|
140 | var lockCount = new object();
|
---|
141 | var random = new FastRandom(1337);
|
---|
142 |
|
---|
143 | Expression bestProgram = null;
|
---|
144 | var config = new PushConfiguration { EvalPushLimit = 4096 };
|
---|
145 | var pool = new PushInterpreterPool(config);
|
---|
146 |
|
---|
147 | var instance = new BenchmarkSuiteInstanceProvider();
|
---|
148 | var data = instance.LoadData(descriptor);
|
---|
149 | var provider = new ManagedPoolProvider<PushProgram>(1024);
|
---|
150 | provider.InitDummyPartition(() => new PushProgram());
|
---|
151 |
|
---|
152 | Parallel.For(0, iterations, i => {
|
---|
153 | var execCounter = 0;
|
---|
154 |
|
---|
155 | var pushProgramPool = provider.CreatePool();
|
---|
156 | var program = CodeGenerator.RandomProgram(pushProgramPool, maxProgramSizeLimit, random);
|
---|
157 |
|
---|
158 | var results = new double[data.OriginalTrainingCount];
|
---|
159 |
|
---|
160 | using (var interpreter = pool.GetInstance(random)) {
|
---|
161 | interpreter.PushProgramPool = pushProgramPool;
|
---|
162 |
|
---|
163 | for (var j = 0; j < data.OriginalTrainingCount; j++) {
|
---|
164 | var example = data.Examples[i];
|
---|
165 |
|
---|
166 | interpreter.BooleanStack.Push(example.InputBoolean);
|
---|
167 | interpreter.IntegerStack.Push(example.InputInt);
|
---|
168 | interpreter.FloatStack.Push(example.InputFloat);
|
---|
169 |
|
---|
170 | interpreter.Run(program);
|
---|
171 | pushProgramPool.Dispose();
|
---|
172 |
|
---|
173 | var diff = GetDiff(example.OutputInt, interpreter.IntegerStack)
|
---|
174 | + GetDiff(example.OutputFloat, interpreter.FloatStack)
|
---|
175 | + GetDiff(example.OutputBoolean, interpreter.BooleanStack);
|
---|
176 |
|
---|
177 | results[j] = diff;
|
---|
178 |
|
---|
179 | execCounter += interpreter.ExecCounter;
|
---|
180 | interpreter.Clear();
|
---|
181 | pushProgramPool = provider.CreatePool();
|
---|
182 | }
|
---|
183 | }
|
---|
184 |
|
---|
185 | lock (lockCount) {
|
---|
186 | globalExecCounter += execCounter;
|
---|
187 | }
|
---|
188 |
|
---|
189 | var avg = results.Average();
|
---|
190 |
|
---|
191 | if (avg >= best) return;
|
---|
192 |
|
---|
193 | lock (lockObj) {
|
---|
194 | if (avg < best) {
|
---|
195 | best = avg;
|
---|
196 | bestProgram = program;
|
---|
197 | }
|
---|
198 | }
|
---|
199 | });
|
---|
200 |
|
---|
201 | var resultsTest = new double[data.OriginalTestCount];
|
---|
202 | Parallel.For(data.OriginalTestCount, data.OriginalTestCount, i => {
|
---|
203 | using (var interpreter = pool.GetInstance()) {
|
---|
204 | var example = data.Examples[i];
|
---|
205 |
|
---|
206 | interpreter.BooleanStack.Push(example.InputBoolean);
|
---|
207 | interpreter.IntegerStack.Push(example.InputInt);
|
---|
208 | interpreter.FloatStack.Push(example.InputFloat);
|
---|
209 |
|
---|
210 | interpreter.Run(bestProgram);
|
---|
211 |
|
---|
212 | var diff = GetDiff(example.OutputInt, interpreter.IntegerStack) +
|
---|
213 | GetDiff(example.OutputFloat, interpreter.FloatStack) +
|
---|
214 | GetDiff(example.OutputBoolean, interpreter.BooleanStack);
|
---|
215 |
|
---|
216 | resultsTest[i] = diff;
|
---|
217 | }
|
---|
218 | });
|
---|
219 |
|
---|
220 | var averageTestResult = resultsTest.Average();
|
---|
221 |
|
---|
222 | Console.WriteLine("Best Training: {0}", best);
|
---|
223 | Console.WriteLine("Test: {0}", averageTestResult);
|
---|
224 | Console.WriteLine("Best trainig program: {0}", bestProgram);
|
---|
225 | Console.WriteLine("ExecCounter: {0}", globalExecCounter);
|
---|
226 | }
|
---|
227 |
|
---|
228 | private static double GetDiff<T>(IReadOnlyList<T> estimated, IStack<T> resultStack)
|
---|
229 | where T : IComparable {
|
---|
230 | var count = Math.Min(estimated.Count, resultStack.Count);
|
---|
231 | var result = resultStack.Peek(count);
|
---|
232 | var comparableLength = Math.Min(estimated.Count, result.Length);
|
---|
233 | var diff = 0d;
|
---|
234 |
|
---|
235 | for (var i = 0; i < comparableLength; i++) {
|
---|
236 | diff += Math.Abs(estimated[i].CompareTo(result[i]));
|
---|
237 | }
|
---|
238 |
|
---|
239 | if (estimated.Count > result.Length) {
|
---|
240 | for (var i = comparableLength; i < estimated.Count; i++) {
|
---|
241 | diff += Math.Abs(estimated[i].CompareTo(default(T)));
|
---|
242 | }
|
---|
243 | }
|
---|
244 |
|
---|
245 | return diff;
|
---|
246 | }
|
---|
247 | }
|
---|
248 | } |
---|