[15289] | 1 | namespace HeuristicLab.Problems.ProgramSynthesis.Push.Evaluator {
|
---|
| 2 | using System;
|
---|
| 3 | using System.Collections.Generic;
|
---|
| 4 | using System.Globalization;
|
---|
[15334] | 5 | using System.Linq;
|
---|
[15289] | 6 |
|
---|
| 7 | using HeuristicLab.BenchmarkSuite;
|
---|
| 8 | using HeuristicLab.Common;
|
---|
| 9 | using HeuristicLab.Core;
|
---|
| 10 | using HeuristicLab.Parameters;
|
---|
| 11 | using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
|
---|
[15334] | 12 | using HeuristicLab.Problems.ProgramSynthesis.Base.Extensions;
|
---|
[15289] | 13 | using HeuristicLab.Problems.ProgramSynthesis.Push.Configuration;
|
---|
| 14 | using HeuristicLab.Problems.ProgramSynthesis.Push.Expressions;
|
---|
| 15 | using HeuristicLab.Problems.ProgramSynthesis.Push.Interpreter;
|
---|
| 16 | using HeuristicLab.Problems.ProgramSynthesis.Push.Problem;
|
---|
| 17 | using HeuristicLab.Problems.ProgramSynthesis.Push.Stack;
|
---|
| 18 |
|
---|
| 19 | [StorableClass]
|
---|
| 20 | public class PushBenchmarkSuiteEvaluator : ParameterizedNamedItem, IPushEvaluator {
|
---|
| 21 |
|
---|
[15334] | 22 | private const string DATA_BOUNDS_PARAMETER_NAME = "DataBounds";
|
---|
| 23 | private const string DATA_PARAMETER_NAME = "Data";
|
---|
| 24 | private const string DATA_PARAMETER_DESCRIPTION = "Program Synthesis";
|
---|
[15289] | 25 |
|
---|
| 26 | public PushBenchmarkSuiteEvaluator() {
|
---|
[15334] | 27 | Parameters.Add(new FixedValueParameter<DataBounds>(DATA_BOUNDS_PARAMETER_NAME));
|
---|
[15289] | 28 |
|
---|
| 29 | Parameters.Add(new ValueParameter<ProblemData>(
|
---|
[15334] | 30 | DATA_PARAMETER_NAME,
|
---|
| 31 | DATA_PARAMETER_DESCRIPTION));
|
---|
[15289] | 32 | }
|
---|
| 33 |
|
---|
| 34 | public PushBenchmarkSuiteEvaluator(ProblemData data) : this() {
|
---|
| 35 | LoadData(data);
|
---|
| 36 | }
|
---|
| 37 |
|
---|
| 38 | [StorableConstructor]
|
---|
| 39 | public PushBenchmarkSuiteEvaluator(bool deserializing) : base(deserializing) { }
|
---|
| 40 |
|
---|
| 41 | public PushBenchmarkSuiteEvaluator(PushBenchmarkSuiteEvaluator origin, Cloner cloner) : base(origin, cloner) {
|
---|
| 42 | Data = cloner.Clone(origin.Data);
|
---|
| 43 | }
|
---|
| 44 |
|
---|
| 45 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
| 46 | return new PushBenchmarkSuiteEvaluator(this, cloner);
|
---|
| 47 | }
|
---|
| 48 |
|
---|
| 49 | public IValueParameter<ProblemData> DataParameter
|
---|
| 50 | {
|
---|
[15334] | 51 | get { return (IValueParameter<ProblemData>)Parameters[DATA_PARAMETER_NAME]; }
|
---|
[15289] | 52 | }
|
---|
| 53 |
|
---|
| 54 | public ProblemData Data
|
---|
| 55 | {
|
---|
| 56 | get { return DataParameter.Value; }
|
---|
| 57 | set { DataParameter.Value = value; }
|
---|
| 58 | }
|
---|
| 59 |
|
---|
| 60 | public IValueParameter<DataBounds> DataBoundsParameter
|
---|
| 61 | {
|
---|
[15334] | 62 | get { return (IValueParameter<DataBounds>)Parameters[DATA_BOUNDS_PARAMETER_NAME]; }
|
---|
[15289] | 63 | }
|
---|
| 64 |
|
---|
| 65 | public DataBounds DataBounds
|
---|
| 66 | {
|
---|
| 67 | get { return DataBoundsParameter.Value; }
|
---|
| 68 | set { DataBoundsParameter.Value = value; }
|
---|
| 69 | }
|
---|
| 70 |
|
---|
| 71 | public void LoadData(ProblemData data) {
|
---|
| 72 | Data = data;
|
---|
| 73 |
|
---|
| 74 | DataBounds.TrainingRange.Start = 0;
|
---|
| 75 | DataBounds.TrainingRange.End = Data.TrainingCount;
|
---|
| 76 | DataBounds.TestRange.Start = Data.TrainingCount;
|
---|
| 77 | DataBounds.TestRange.End = Data.TrainingCount + Data.TestCount;
|
---|
| 78 | }
|
---|
| 79 |
|
---|
| 80 | public EvaluationResult Evaluate(IPushInterpreter interpreter, PushProgram program, int start, int end) {
|
---|
| 81 | var length = end - start;
|
---|
| 82 | if (length <= 0) return null;
|
---|
| 83 |
|
---|
| 84 | var evaluationResult = new EvaluationResult(length);
|
---|
| 85 |
|
---|
| 86 | for (var i = start; i < end; i++) {
|
---|
| 87 | var result = Evaluate(interpreter, program, i);
|
---|
| 88 | evaluationResult.ExampleQualities[i - start] = result;
|
---|
| 89 | evaluationResult.AvgQuality += result;
|
---|
| 90 | interpreter.Reset();
|
---|
| 91 | }
|
---|
| 92 |
|
---|
| 93 | evaluationResult.AvgQuality /= length;
|
---|
| 94 |
|
---|
| 95 | return evaluationResult;
|
---|
| 96 | }
|
---|
| 97 |
|
---|
| 98 | public EvaluationResult EvaluateTest(IPushInterpreter interpreter, PushProgram program) {
|
---|
| 99 | return Evaluate(interpreter, program, DataBounds.TestRange.Start, DataBounds.TestRange.End);
|
---|
| 100 | }
|
---|
| 101 |
|
---|
| 102 | public EvaluationResult EvaluateTest(IReadOnlyPushConfiguration config, PushProgram program, IRandom random) {
|
---|
| 103 | var interpreter = new PushInterpreter(config, random);
|
---|
| 104 | return EvaluateTest(interpreter, program);
|
---|
| 105 | }
|
---|
| 106 |
|
---|
| 107 | public EvaluationResult EvaluateTest(PushInterpreterPool pool, PushProgram program, IRandom random) {
|
---|
| 108 | using (var interpreter = pool.Create(random)) {
|
---|
| 109 | return EvaluateTest(interpreter, program);
|
---|
| 110 | }
|
---|
| 111 | }
|
---|
| 112 |
|
---|
| 113 | public EvaluationResult EvaluateTraining(IPushInterpreter interpreter, PushProgram program) {
|
---|
| 114 | return Evaluate(interpreter, program, DataBounds.TrainingRange.Start, DataBounds.TrainingRange.End);
|
---|
| 115 | }
|
---|
| 116 |
|
---|
| 117 | public EvaluationResult EvaluateTraining(IReadOnlyPushConfiguration config, PushProgram program, IRandom random) {
|
---|
| 118 | var interpreter = new PushInterpreter(config, random);
|
---|
| 119 | return EvaluateTraining(interpreter, program);
|
---|
| 120 | }
|
---|
| 121 |
|
---|
| 122 | public EvaluationResult EvaluateTraining(PushInterpreterPool pool, PushProgram program, IRandom random) {
|
---|
| 123 | using (var interpreter = pool.Create(random)) {
|
---|
| 124 | return EvaluateTraining(interpreter, program);
|
---|
| 125 | }
|
---|
| 126 | }
|
---|
| 127 |
|
---|
| 128 | public double Evaluate(IPushInterpreter interpreter, PushProgram program, int exampleIndex) {
|
---|
| 129 | var example = Data.Examples[exampleIndex];
|
---|
| 130 |
|
---|
| 131 | interpreter.InitExample(example);
|
---|
| 132 | interpreter.Run(program);
|
---|
| 133 |
|
---|
| 134 | switch (Data.ProblemType) {
|
---|
[15334] | 135 | // special
|
---|
[15289] | 136 | case ProblemType.NumberIO: return NumberIo(interpreter, example);
|
---|
[15334] | 137 | case ProblemType.StringDifferences: return StringDifferences(interpreter, example);
|
---|
| 138 | case ProblemType.EvenSquares: return EvenSquares(interpreter, example);
|
---|
| 139 | case ProblemType.WallisPi: return WallisPi(interpreter, example);
|
---|
| 140 | case ProblemType.VectorSummed: return VectorsSummed(interpreter, example);
|
---|
| 141 | case ProblemType.XWordLines: return XWordLines(interpreter, example);
|
---|
| 142 | case ProblemType.NegativeToZero: return NegativeToZero(interpreter, example);
|
---|
| 143 | case ProblemType.WordStats: return WordStats(interpreter, example);
|
---|
| 144 | case ProblemType.Checksum: return Checksum(interpreter, example);
|
---|
| 145 | case ProblemType.Grades: return Grade(interpreter, example);
|
---|
| 146 | case ProblemType.Syllables: return Syllables(interpreter, example);
|
---|
| 147 |
|
---|
| 148 | // printed string right/wrong
|
---|
| 149 | case ProblemType.Smallest:
|
---|
| 150 | case ProblemType.Median: return PrintedStringRight(interpreter, example);
|
---|
[15345] | 151 | case ProblemType.MedianIntegerError: return MedianIntegerError(interpreter, example);
|
---|
[15334] | 152 |
|
---|
| 153 | // boolean error
|
---|
| 154 | case ProblemType.MirrorImage:
|
---|
| 155 | case ProblemType.SuperAnagrams:
|
---|
| 156 | case ProblemType.CompareStringLengths: return GetDiff(example.OutputBoolean, interpreter.BooleanStack, Data.WorstResult, BooleanDiffer);
|
---|
| 157 |
|
---|
| 158 | // float error
|
---|
[15341] | 159 | case ProblemType.VectorAverage: return GetDiff(example.OutputFloat, interpreter.FloatStack, Data.WorstResult, (a, b) => FloatDiffer(a, b, example.OutputFloatPrecision, Data.WorstResult));
|
---|
[15334] | 160 |
|
---|
| 161 | // integer error
|
---|
| 162 | case ProblemType.LastIndexOfZero:
|
---|
| 163 | case ProblemType.CountOdds:
|
---|
| 164 | case ProblemType.SumOfSquares:
|
---|
| 165 | case ProblemType.ScrabbleScore:
|
---|
| 166 | case ProblemType.CollatzNumbers: return GetDiff(example.OutputInteger, interpreter.IntegerStack, Data.WorstResult, IntegerDiffer);
|
---|
| 167 |
|
---|
| 168 | // levenshtein distance + integer error
|
---|
| 169 | case ProblemType.ReplaceSpaceWithNewline:
|
---|
| 170 | return GetPrintDiffer(example.OutputPrint, interpreter.PrintStack, Data.WorstResult)
|
---|
| 171 | + GetDiff(example.OutputInteger, interpreter.IntegerStack, Data.WorstResult, IntegerDiffer);
|
---|
| 172 |
|
---|
| 173 | // printed levenshtein distance
|
---|
| 174 | case ProblemType.ForLoopIndex:
|
---|
| 175 | case ProblemType.DoubleLetters:
|
---|
| 176 | case ProblemType.StringLengthsBackwards:
|
---|
| 177 | case ProblemType.PigLatin:
|
---|
| 178 | case ProblemType.Digits:
|
---|
| 179 | case ProblemType.SmallOrLarge: return GetPrintDiffer(example.OutputPrint, interpreter.PrintStack, Data.WorstResult);
|
---|
[15289] | 180 | }
|
---|
| 181 |
|
---|
| 182 | return Default(interpreter, example);
|
---|
| 183 | }
|
---|
| 184 |
|
---|
| 185 | private double Default(IPushInterpreter interpreter, Example example) {
|
---|
| 186 | var integerDiff = GetDiff(example.OutputInteger, interpreter.IntegerStack, Data.WorstResult, IntegerDiffer);
|
---|
[15341] | 187 | var floatDiff = GetDiff(example.OutputFloat, interpreter.FloatStack, Data.WorstResult, (a, b) => FloatDiffer(a, b, example.OutputFloatPrecision, Data.WorstResult));
|
---|
[15289] | 188 | var booleanDiff = GetDiff(example.OutputBoolean, interpreter.BooleanStack, Data.WorstResult, BooleanDiffer);
|
---|
| 189 | var stringDiff = GetDiff(example.OutputString, interpreter.StringStack, Data.WorstResult, StringDiffer);
|
---|
| 190 | var charDiff = GetDiff(example.OutputChar, interpreter.CharStack, Data.WorstResult, CharDiffer);
|
---|
| 191 | var printDiff = GetPrintDiffer(example.OutputPrint, interpreter.PrintStack, Data.WorstResult);
|
---|
| 192 | var integerVectorDiff = GetVectorDiff(example.OutputIntegerVector, interpreter.IntegerVectorStack, Data.WorstResult, (a, b) => VectorDiffer(a, b, IntegerDiffer));
|
---|
[15341] | 193 | var floatVectorDiff = GetVectorDiff(example.OutputFloatVector, interpreter.FloatVectorStack, Data.WorstResult, (a, b) => VectorDiffer(a, b, (x, y) => FloatDiffer(x, y, example.OutputFloatVectorPrecision, Data.WorstResult)));
|
---|
[15289] | 194 | var stringVectorDiff = GetVectorDiff(example.OutputStringVector, interpreter.StringVectorStack, Data.WorstResult, (a, b) => VectorDiffer(a, b, StringDiffer));
|
---|
| 195 |
|
---|
| 196 | var result =
|
---|
| 197 | integerDiff
|
---|
| 198 | + floatDiff
|
---|
| 199 | + booleanDiff
|
---|
| 200 | + stringDiff
|
---|
| 201 | + charDiff
|
---|
| 202 | + printDiff
|
---|
| 203 | + integerVectorDiff
|
---|
| 204 | + floatVectorDiff
|
---|
| 205 | + stringVectorDiff;
|
---|
| 206 |
|
---|
| 207 | return result;
|
---|
| 208 | }
|
---|
| 209 |
|
---|
[15334] | 210 | private double PrintedStringRight(IPushInterpreter interpreter, Example example) {
|
---|
| 211 | return interpreter.PrintStack.IsEmpty ||
|
---|
| 212 | !interpreter.PrintStack.ToString().Equals(example.OutputPrint)
|
---|
| 213 | ? 1
|
---|
| 214 | : 0;
|
---|
[15289] | 215 | }
|
---|
| 216 |
|
---|
[15345] | 217 | private double MedianIntegerError(IPushInterpreter interpreter, Example example) {
|
---|
| 218 | var printResult = interpreter.PrintStack.ToString();
|
---|
| 219 |
|
---|
| 220 | long result;
|
---|
| 221 | if (long.TryParse(printResult, out result)) {
|
---|
| 222 | var diff = Math.Abs(result - example.OutputInteger[0]);
|
---|
| 223 |
|
---|
| 224 | return Math.Min(Data.WorstResult, diff);
|
---|
| 225 | }
|
---|
| 226 |
|
---|
| 227 | return Data.WorstResult;
|
---|
| 228 | }
|
---|
| 229 |
|
---|
[15289] | 230 | private double NumberIo(IPushInterpreter interpreter, Example example) {
|
---|
[15334] | 231 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty)
|
---|
[15289] | 232 | return Data.WorstResult;
|
---|
| 233 |
|
---|
| 234 | var printResult = interpreter.PrintStack.ToString();
|
---|
| 235 |
|
---|
| 236 | double value;
|
---|
| 237 | if (double.TryParse(printResult, NumberStyles.Number | NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out value)) {
|
---|
| 238 | var diff = Math.Abs(example.OutputFloat[0] - value);
|
---|
[15334] | 239 | diff = Math.Min(diff, Data.WorstResult);
|
---|
[15289] | 240 |
|
---|
[15334] | 241 | return diff;
|
---|
[15289] | 242 | }
|
---|
| 243 |
|
---|
| 244 | var penaltyError = Data.WorstResult / 4.0d;
|
---|
| 245 | var levenshteinDistance = GetPrintDiffer(
|
---|
| 246 | example.OutputPrint,
|
---|
| 247 | printResult,
|
---|
| 248 | Data.WorstResult);
|
---|
| 249 |
|
---|
[15334] | 250 | var result = levenshteinDistance + penaltyError;
|
---|
| 251 | return Math.Min(result, Data.WorstResult);
|
---|
[15289] | 252 | }
|
---|
| 253 |
|
---|
[15334] | 254 | private double StringDifferences(IPushInterpreter interpreter, Example example) {
|
---|
| 255 | if (string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty) {
|
---|
| 256 | return 0.0;
|
---|
| 257 | }
|
---|
| 258 |
|
---|
| 259 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty) {
|
---|
| 260 | return Data.WorstResult;
|
---|
| 261 | }
|
---|
| 262 |
|
---|
| 263 | var printResult = interpreter.PrintStack.ToString();
|
---|
| 264 | var distance = example.OutputPrint.LevenshteinDistance(printResult);
|
---|
| 265 | var lineCountWithCorrectFormat = interpreter.PrintStack.AsStrings().Count(line => {
|
---|
| 266 | var parts = line.Split(' ');
|
---|
| 267 |
|
---|
| 268 | if (parts.Length != 3)
|
---|
| 269 | return false;
|
---|
| 270 |
|
---|
| 271 | var part0 = parts[0].Trim();
|
---|
| 272 | if (part0.Length == 0 || !part0.IsNumeric())
|
---|
| 273 | return false;
|
---|
| 274 |
|
---|
| 275 | var part1 = parts[1].Trim();
|
---|
| 276 | if (part1.Length != 1)
|
---|
| 277 | return false;
|
---|
| 278 |
|
---|
| 279 | var part2 = parts[2].Trim();
|
---|
| 280 | if (part2.Length != 1)
|
---|
| 281 | return false;
|
---|
| 282 |
|
---|
| 283 | return true;
|
---|
| 284 | });
|
---|
| 285 |
|
---|
| 286 | var lineCountWithInvalidFormat = example.OutputPrintLineCount - lineCountWithCorrectFormat;
|
---|
| 287 | lineCountWithInvalidFormat = Math.Min(lineCountWithInvalidFormat, example.OutputPrintLineCount);
|
---|
| 288 |
|
---|
| 289 | var result = distance + lineCountWithInvalidFormat;
|
---|
| 290 | return Math.Min(result, Data.WorstResult);
|
---|
| 291 | }
|
---|
| 292 |
|
---|
| 293 | private double EvenSquares(IPushInterpreter interpreter, Example example) {
|
---|
| 294 | if (string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty) {
|
---|
| 295 | return 0.0;
|
---|
| 296 | }
|
---|
| 297 |
|
---|
| 298 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty) {
|
---|
| 299 | return Data.WorstResult;
|
---|
| 300 | }
|
---|
| 301 |
|
---|
| 302 | var distance = example.OutputPrint.LevenshteinDistance(interpreter.PrintStack.ToString());
|
---|
| 303 | var printLines = interpreter.PrintStack.AsStrings().ToArray();
|
---|
| 304 | var lineCountWithCorrectFormat = printLines.Count(line => line.IsNumeric());
|
---|
[15345] | 305 | var lineCountWithInvalidFormat = Math.Max(0, example.OutputPrintLineCount - lineCountWithCorrectFormat);
|
---|
[15334] | 306 |
|
---|
| 307 | lineCountWithInvalidFormat = Math.Abs(Math.Min(lineCountWithInvalidFormat, example.OutputPrintLineCount));
|
---|
| 308 |
|
---|
| 309 | var compareLength = Math.Min(printLines.Length, example.OutputPrintLineCount);
|
---|
| 310 | long integerError = 0;
|
---|
| 311 |
|
---|
| 312 | for (var i = 0; i < compareLength; i++) {
|
---|
| 313 | long value;
|
---|
| 314 | if (long.TryParse(printLines[i], out value)) {
|
---|
| 315 | integerError += Math.Abs(value - example.OutputIntegerVector[0][i]);
|
---|
| 316 | } else {
|
---|
| 317 | integerError += example.OutputPrintLineCount > 0
|
---|
| 318 | ? (long)(Data.WorstResult / example.OutputPrintLineCount)
|
---|
| 319 | : 0;
|
---|
| 320 | }
|
---|
| 321 | }
|
---|
| 322 |
|
---|
| 323 | var result = distance + lineCountWithInvalidFormat + integerError;
|
---|
| 324 | return Math.Min(result, Data.WorstResult);
|
---|
| 325 | }
|
---|
| 326 |
|
---|
| 327 | private double WallisPi(IPushInterpreter interpreter, Example example) {
|
---|
| 328 | if (interpreter.FloatStack.IsEmpty)
|
---|
| 329 | return Data.WorstResult;
|
---|
| 330 |
|
---|
[15341] | 331 | var floatDiff = GetDiff(example.OutputFloat, interpreter.FloatStack, Data.WorstResult, (a, b) => FloatDiffer(a, b, example.OutputFloatPrecision, Data.WorstResult));
|
---|
[15334] | 332 | var expectedStr = example.OutputFloat[0].ToString(Data.FloatStringFormat, CultureInfo.CurrentCulture);
|
---|
| 333 | var resultStr = interpreter.FloatStack.TopOrDefault.ToString(Data.FloatStringFormat, CultureInfo.CurrentCulture);
|
---|
| 334 | var distance = expectedStr.LevenshteinDistance(resultStr);
|
---|
| 335 |
|
---|
| 336 | var result = floatDiff + distance;
|
---|
| 337 | return Math.Min(result, Data.WorstResult);
|
---|
| 338 | }
|
---|
| 339 |
|
---|
| 340 | private double VectorsSummed(IPushInterpreter interpreter, Example example) {
|
---|
| 341 | if (interpreter.IntegerVectorStack.IsEmpty && example.OutputIntegerVector.Length > 0)
|
---|
| 342 | return Data.WorstResult;
|
---|
| 343 |
|
---|
| 344 | var resultVector = interpreter.IntegerVectorStack[0];
|
---|
| 345 | var expectedVector = example.OutputIntegerVector[0];
|
---|
| 346 | var comparableLength = Math.Min(resultVector.Count, expectedVector.Length);
|
---|
| 347 |
|
---|
| 348 | // add penalty if length does not match
|
---|
| 349 | var result = expectedVector.Length > 0
|
---|
| 350 | ? (expectedVector.Length - comparableLength) * (Data.WorstResult / expectedVector.Length)
|
---|
| 351 | : 0;
|
---|
| 352 |
|
---|
| 353 | for (var i = 0; i < comparableLength; i++) {
|
---|
| 354 | result += Math.Abs(resultVector[i] - expectedVector[i]);
|
---|
| 355 | }
|
---|
| 356 |
|
---|
| 357 | return Math.Min(result, Data.WorstResult);
|
---|
| 358 | }
|
---|
| 359 |
|
---|
| 360 | private double XWordLines(IPushInterpreter interpreter, Example example) {
|
---|
| 361 | if (string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty) {
|
---|
| 362 | return 0.0;
|
---|
| 363 | }
|
---|
| 364 |
|
---|
| 365 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty)
|
---|
| 366 | return Data.WorstResult;
|
---|
| 367 |
|
---|
| 368 | var estimatedWordsPerLineCount = example.InputInteger[0];
|
---|
| 369 | var estimatedWordCount = example.OutputInteger[0];
|
---|
| 370 | var estimatedLastWordCount = estimatedWordCount - estimatedWordsPerLineCount * Math.Max(0, example.OutputPrintLineCount - 1);
|
---|
| 371 | var printResult = interpreter.PrintStack.ToString();
|
---|
| 372 | var distance = example.OutputPrint.LevenshteinDistance(printResult);
|
---|
| 373 | var lineCount = interpreter.PrintStack.Count;
|
---|
| 374 | var lineCountError = Math.Abs(example.OutputPrintLineCount - lineCount);
|
---|
| 375 | var totalWordsPerLineError = 0L;
|
---|
| 376 |
|
---|
| 377 | for (var i = 0; i < interpreter.PrintStack.Lines.Count - 1; i++) {
|
---|
| 378 | totalWordsPerLineError += Math.Abs(interpreter.PrintStack.Lines[i].Split(' ').Length - estimatedWordsPerLineCount);
|
---|
| 379 | }
|
---|
| 380 |
|
---|
| 381 | // last line
|
---|
| 382 | var lastLine = interpreter.PrintStack.Lines[interpreter.PrintStack.Lines.Count - 1];
|
---|
| 383 | totalWordsPerLineError += Math.Abs(lastLine.Split(' ').Length - estimatedLastWordCount);
|
---|
| 384 |
|
---|
| 385 | var result = distance + lineCountError + totalWordsPerLineError;
|
---|
| 386 | return Math.Min(result, Data.WorstResult);
|
---|
| 387 | }
|
---|
| 388 |
|
---|
| 389 | private double NegativeToZero(IPushInterpreter interpreter, Example example) {
|
---|
| 390 | if (interpreter.IntegerVectorStack.IsEmpty && example.OutputIntegerVector.Length > 0)
|
---|
| 391 | return Data.WorstResult;
|
---|
| 392 |
|
---|
| 393 | var resultVector = interpreter.IntegerVectorStack.Top;
|
---|
| 394 | var expectedVector = example.OutputIntegerVector[0];
|
---|
| 395 | var comparableLength = Math.Min(resultVector.Count, expectedVector.Length);
|
---|
| 396 |
|
---|
| 397 | // add penalty if length does not match
|
---|
| 398 | var result = expectedVector.Length > 0
|
---|
| 399 | ? (expectedVector.Length - comparableLength) * (Data.WorstResult / expectedVector.Length)
|
---|
| 400 | : 0;
|
---|
| 401 |
|
---|
| 402 | for (var i = 0; i < comparableLength; i++) {
|
---|
| 403 | var expectedStr = expectedVector[i].ToString();
|
---|
| 404 | var resultStr = resultVector[i].ToString();
|
---|
| 405 | result += expectedStr.LevenshteinDistance(resultStr);
|
---|
| 406 | }
|
---|
| 407 |
|
---|
| 408 | return Math.Min(result, Data.WorstResult);
|
---|
| 409 | }
|
---|
| 410 |
|
---|
| 411 | private double WordStats(IPushInterpreter interpreter, Example example) {
|
---|
| 412 | if (string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty) {
|
---|
| 413 | return 0.0;
|
---|
| 414 | }
|
---|
| 415 |
|
---|
| 416 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty)
|
---|
| 417 | return Data.WorstResult;
|
---|
| 418 |
|
---|
| 419 | var printStr = interpreter.PrintStack.ToString();
|
---|
| 420 | var distance = example.OutputPrint.LevenshteinDistance(printStr);
|
---|
| 421 |
|
---|
| 422 | var statsError = 0.0;
|
---|
| 423 | long numberOfSentences;
|
---|
| 424 | var expectedNumberOfSentences = example.OutputInteger[0];
|
---|
| 425 | statsError += BenchmarkSuite.Problems.WordStats.GetNumberOfSentences(printStr, out numberOfSentences)
|
---|
| 426 | ? Math.Abs(expectedNumberOfSentences - numberOfSentences)
|
---|
| 427 | : Data.WorstResult / 3.0;
|
---|
| 428 |
|
---|
| 429 | double averageSentenceLength;
|
---|
| 430 | var expectedAverageSentenceLength = example.OutputFloat[0];
|
---|
| 431 | statsError += BenchmarkSuite.Problems.WordStats.GetAverageSentenceLength(printStr, out averageSentenceLength)
|
---|
| 432 | ? Math.Abs(expectedAverageSentenceLength - averageSentenceLength)
|
---|
| 433 | : Data.WorstResult / 3.0;
|
---|
| 434 |
|
---|
| 435 | var result = distance + statsError;
|
---|
| 436 | return Math.Min(result, Data.WorstResult);
|
---|
| 437 | }
|
---|
| 438 |
|
---|
| 439 | private double Checksum(IPushInterpreter interpreter, Example example) {
|
---|
| 440 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty)
|
---|
| 441 | return Data.WorstResult;
|
---|
| 442 |
|
---|
| 443 | var printStr = interpreter.PrintStack.ToString();
|
---|
| 444 | var distance = example.OutputPrint.LevenshteinDistance(printStr);
|
---|
| 445 |
|
---|
| 446 | var expectedLastCharIndex = example.OutputPrint.Length - 1;
|
---|
| 447 | var result = distance;
|
---|
| 448 |
|
---|
| 449 | if (expectedLastCharIndex >= 0) {
|
---|
| 450 | var expectedLastChar = example.OutputPrint[expectedLastCharIndex];
|
---|
| 451 | var resultLastCharIndex = printStr.Length - 1;
|
---|
| 452 |
|
---|
| 453 | if (resultLastCharIndex >= 0) {
|
---|
| 454 | var resultLastChar = printStr[resultLastCharIndex];
|
---|
| 455 | result += Math.Abs(expectedLastChar - resultLastChar);
|
---|
| 456 | }
|
---|
| 457 | }
|
---|
| 458 |
|
---|
| 459 | return Math.Min(result, Data.WorstResult);
|
---|
| 460 | }
|
---|
| 461 |
|
---|
| 462 | private double Syllables(IPushInterpreter interpreter, Example example) {
|
---|
| 463 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty)
|
---|
| 464 | return Data.WorstResult;
|
---|
| 465 |
|
---|
| 466 | var printStr = interpreter.PrintStack.ToString();
|
---|
| 467 | var distance = example.OutputPrint.LevenshteinDistance(printStr);
|
---|
| 468 |
|
---|
| 469 | long numberOfSyllables;
|
---|
| 470 | var expectedNumberOfSyllables = example.OutputInteger[0];
|
---|
| 471 | var numberOfSyllablesError = BenchmarkSuite.Problems.Syllables.GetNumberOfSyllables(printStr, out numberOfSyllables)
|
---|
| 472 | ? Math.Abs(expectedNumberOfSyllables - numberOfSyllables)
|
---|
| 473 | : Data.WorstResult / 2.0;
|
---|
| 474 |
|
---|
| 475 | var result = distance + numberOfSyllablesError;
|
---|
| 476 | return Math.Min(result, Data.WorstResult);
|
---|
| 477 | }
|
---|
| 478 |
|
---|
| 479 | private double Grade(IPushInterpreter interpreter, Example example) {
|
---|
| 480 | if (!string.IsNullOrEmpty(example.OutputPrint) && interpreter.PrintStack.IsEmpty)
|
---|
| 481 | return Data.WorstResult;
|
---|
| 482 |
|
---|
| 483 | var printStr = interpreter.PrintStack.ToString();
|
---|
| 484 | var distance = example.OutputPrint.LevenshteinDistance(printStr);
|
---|
| 485 |
|
---|
| 486 | char grade;
|
---|
| 487 | var expectedGrade = example.OutputChar[0];
|
---|
| 488 | var gradeError = BenchmarkSuite.Problems.Grades.GetGrade(printStr, out grade)
|
---|
| 489 | ? Math.Abs(expectedGrade - grade)
|
---|
| 490 | : Data.WorstResult / 2;
|
---|
| 491 |
|
---|
| 492 | var result = distance + gradeError;
|
---|
| 493 | return Math.Min(result, Data.WorstResult);
|
---|
| 494 | }
|
---|
| 495 |
|
---|
[15341] | 496 | private static double FloatDiffer(double a, double b, int digits, double worst) {
|
---|
[15289] | 497 | var result = Math.Round(a, digits) - Math.Round(b, digits);
|
---|
| 498 |
|
---|
| 499 | // ReSharper disable once CompareOfFloatsByEqualityOperator
|
---|
[15341] | 500 | if (result == double.MinValue || double.IsPositiveInfinity(result) ||
|
---|
| 501 | double.IsNaN(result) ||
|
---|
| 502 | result == double.MaxValue || double.IsNegativeInfinity(result))
|
---|
| 503 | return worst;
|
---|
[15289] | 504 |
|
---|
| 505 | return Math.Abs(result);
|
---|
| 506 | }
|
---|
| 507 |
|
---|
| 508 | private static double IntegerDiffer(long a, long b) {
|
---|
| 509 | var result = a - b;
|
---|
| 510 |
|
---|
| 511 | return result == long.MinValue ? long.MaxValue : Math.Abs(result);
|
---|
| 512 | }
|
---|
| 513 |
|
---|
| 514 | private static double BooleanDiffer(bool a, bool b) {
|
---|
| 515 | return a == b ? 0 : 1;
|
---|
| 516 | }
|
---|
| 517 |
|
---|
| 518 | private static double StringDiffer(string a, string b) {
|
---|
[15334] | 519 | return a.LevenshteinDistance(b);
|
---|
[15289] | 520 | }
|
---|
| 521 |
|
---|
| 522 | private static double CharDiffer(char a, char b) {
|
---|
| 523 | return IntegerDiffer(a, b);
|
---|
| 524 | }
|
---|
| 525 |
|
---|
| 526 | private static double VectorDiffer<T>(IReadOnlyList<T> estimated, IReadOnlyList<T> outcome, Func<T, T, double> differ) {
|
---|
| 527 | var length = Math.Min(estimated.Count, outcome.Count);
|
---|
| 528 | var result = 0d;
|
---|
| 529 |
|
---|
| 530 | for (var i = 0; i < length; i++) {
|
---|
| 531 | result += differ(estimated[i], outcome[i]);
|
---|
| 532 | }
|
---|
| 533 |
|
---|
| 534 | if (estimated.Count > outcome.Count) {
|
---|
| 535 | var remainingLength = estimated.Count - length;
|
---|
| 536 | for (var i = 0; i < remainingLength; i++)
|
---|
| 537 | result += differ(estimated[i], default(T));
|
---|
| 538 | }
|
---|
| 539 |
|
---|
| 540 | return result;
|
---|
| 541 | }
|
---|
| 542 |
|
---|
| 543 | private static double GetPrintDiffer(string estimated, string printResult, double worstResult) {
|
---|
[15334] | 544 | var distance = estimated.LevenshteinDistance(printResult);
|
---|
[15289] | 545 |
|
---|
| 546 | return Math.Min(distance, worstResult);
|
---|
| 547 | }
|
---|
| 548 |
|
---|
| 549 | private static double GetPrintDiffer(string estimated, IPrintStack printStack, double worstResult) {
|
---|
| 550 | var printResult = printStack.ToString();
|
---|
[15334] | 551 | var distance = estimated.LevenshteinDistance(printResult);
|
---|
[15289] | 552 |
|
---|
| 553 | return Math.Min(distance, worstResult);
|
---|
| 554 | }
|
---|
| 555 |
|
---|
| 556 | private static double GetVectorDiff<T>(IReadOnlyList<IReadOnlyList<T>> estimated, IPushStack<IReadOnlyList<T>> resultStack, double worstResult, Func<IReadOnlyList<T>, IReadOnlyList<T>, double> differ)
|
---|
| 557 | where T : IComparable {
|
---|
| 558 | if (estimated.Count == 0) return 0d;
|
---|
[15341] | 559 | if (resultStack.IsEmpty && estimated.Count > 0) return worstResult;
|
---|
[15289] | 560 |
|
---|
| 561 | var diff = 0d;
|
---|
| 562 | var comparableLength = Math.Min(estimated.Count, resultStack.Count);
|
---|
| 563 |
|
---|
| 564 | if (!resultStack.IsEmpty) {
|
---|
| 565 |
|
---|
[15341] | 566 | for (var i = 0; i < comparableLength && diff < worstResult; i++) {
|
---|
| 567 | diff += differ(estimated[i], resultStack[i]);
|
---|
[15289] | 568 | }
|
---|
| 569 | }
|
---|
| 570 |
|
---|
| 571 | var emptyTArray = new T[0];
|
---|
[15341] | 572 | for (var i = comparableLength; i < estimated.Count - comparableLength && diff < worstResult; i++) {
|
---|
[15289] | 573 | diff += differ(estimated[i], emptyTArray);
|
---|
| 574 | }
|
---|
| 575 |
|
---|
[15344] | 576 | return double.IsNaN(diff) || double.IsInfinity(diff)
|
---|
| 577 | ? worstResult
|
---|
| 578 | : diff;
|
---|
[15289] | 579 | }
|
---|
| 580 |
|
---|
| 581 | private static double GetDiff<T>(IReadOnlyList<T> estimated, IPushStack<T> resultStack, double worstResult, Func<T, T, double> differ)
|
---|
| 582 | where T : IComparable {
|
---|
| 583 | if (estimated.Count == 0) return 0d;
|
---|
[15341] | 584 | if (resultStack.IsEmpty && estimated.Count > 0) return worstResult;
|
---|
[15289] | 585 |
|
---|
| 586 | var diff = 0d;
|
---|
| 587 | var comparableLength = Math.Min(estimated.Count, resultStack.Count);
|
---|
| 588 |
|
---|
| 589 | if (!resultStack.IsEmpty) {
|
---|
[15341] | 590 | for (var i = 0; i < comparableLength && diff < worstResult; i++) {
|
---|
| 591 | diff += differ(estimated[i], resultStack[i]);
|
---|
[15289] | 592 | }
|
---|
| 593 | }
|
---|
| 594 |
|
---|
[15341] | 595 | for (var i = comparableLength; i < estimated.Count - comparableLength && diff < worstResult; i++) {
|
---|
[15289] | 596 | diff += differ(estimated[i], default(T));
|
---|
| 597 | }
|
---|
| 598 |
|
---|
[15344] | 599 | return double.IsNaN(diff) || double.IsInfinity(diff)
|
---|
| 600 | ? worstResult
|
---|
| 601 | : diff;
|
---|
[15289] | 602 | }
|
---|
| 603 | }
|
---|
| 604 | }
|
---|