Ignore:
Timestamp:
11/09/11 10:59:31 (8 years ago)
Author:
gkronber
Message:

#1081: cleared up definition of accuracy metrics for time series prognosis to make the distinction between one n-step forecast and n one-step forecasts clearer. Implemented calculators for time series accuracy metrics to support the calculation of the average accuracy over m n-step forecasts and adapted the unit tests accordingly.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/sources/HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/StatisticCalculatorsTest.cs

    r6880 r6974  
    2020#endregion
    2121
     22using System;
    2223using System.Collections.Generic;
    2324using System.Linq;
     
    140141      }
    141142    }
     143
     144    [TestMethod]
     145    public void CalculateDirectionalSymmetryTest() {
     146      // delta: +0.01, +1, -0.01, -2, -0.01, -1, +0.01, +2
     147      var original = new double[]
     148                       {
     149                         0,
     150                         0.01,
     151                         1.01,
     152                         1,
     153                         -1,
     154                         -1.01,
     155                         -2.01,
     156                         -2,
     157                         0
     158                       };
     159      // delta to original(t-1): +1, +0, -1, -0, -1, +0.01, +0.01, +2
     160      var estimated = new double[]
     161                        {
     162                          -1,
     163                          1,
     164                          0.01,
     165                          0.01,
     166                          1,
     167                          -1,
     168                          -1.02,
     169                          -2.02,
     170                          0
     171                        };
     172
     173      // one-step forecast
     174      var startValues = original;
     175      var actualContinuations = from x in original.Skip(1)
     176                                select Enumerable.Repeat(x, 1);
     177      var predictedContinuations = from x in estimated.Skip(1)
     178                                   select Enumerable.Repeat(x, 1);
     179      double expected = 0.5;  // half of the predicted deltas are correct
     180      OnlineCalculatorError errorState;
     181      double actual = OnlineDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
     182      Assert.AreEqual(expected, actual, 1E-9);
     183    }
     184    [TestMethod]
     185    public void CalculateWeightedDirectionalSymmetryTest() {
     186      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 }; // +0.01, +1, -0.01, -2, -0.01, -1, +0.01, +2
     187      var estimated = new double[] { 1, 2, 2, 1, 1, 0, 0.01, 0.02, 2.02 }; // delta to original: +2, +1.99, -0.01, 0, +1, -1.02, +2.01, +4.02
     188      // one-step forecast
     189      var startValues = original;
     190      var actualContinuations = from x in original.Skip(1)
     191                                select Enumerable.Repeat(x, 1);
     192      var predictedContinuations = from x in estimated.Skip(1)
     193                                   select Enumerable.Repeat(x, 1);
     194      // absolute errors = 1.99, 0.99, 0, 2, 1.01, 2.02, 2.02, 2.02     
     195      // sum of absolute errors for correctly predicted deltas = 2.97
     196      // sum of absolute errors for incorrectly predicted deltas = 3.03
     197      double expected = 5.03 / 7.02;
     198      OnlineCalculatorError errorState;
     199      double actual = OnlineWeightedDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
     200      Assert.AreEqual(expected, actual, 1E-9);
     201    }
     202    [TestMethod]
     203    public void CalculateTheilsUTest() {
     204      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 };
     205      var estimated = new double[] { 1, 1.01, 0.01, 2, 0, -0.01, -1.01, -3, 1 };
     206      // one-step forecast
     207      var startValues = original;
     208      var actualContinuations = from x in original.Skip(1)
     209                                select Enumerable.Repeat(x, 1);
     210      var predictedContinuations = from x in estimated.Skip(1)
     211                                   select Enumerable.Repeat(x, 1);
     212      // Sum of squared errors of model y(t+1) = y(t) = 10.0004
     213      // Sum of squared errors of predicted values = 8 
     214      double expected = Math.Sqrt(8 / 10.0004);
     215      OnlineCalculatorError errorState;
     216      double actual = OnlineTheilsUStatisticCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
     217      Assert.AreEqual(expected, actual, 1E-9);
     218    }
     219    [TestMethod]
     220    public void CalculateAccuracyTest() {
     221      var original = new double[] { 1, 1, 0, 0 };
     222      var estimated = new double[] { 1, 0, 1, 0 };
     223      double expected = 0.5;
     224      OnlineCalculatorError errorState;
     225      double actual = OnlineAccuracyCalculator.Calculate(original, estimated, out errorState);
     226      Assert.AreEqual(expected, actual, 1E-9);
     227    }
     228
     229    [TestMethod]
     230    public void CalculateMeanAbsolutePercentageErrorTest() {
     231      var original = new double[] { 1, 2, 3, 1, 5 };
     232      var estimated = new double[] { 2, 1, 3, 1, 0 };
     233      double expected = 0.5;
     234      OnlineCalculatorError errorState;
     235      double actual = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(original, estimated, out errorState);
     236      Assert.AreEqual(expected, actual, 1E-9);
     237      Assert.AreEqual(OnlineCalculatorError.None, errorState);
     238
     239      // if the original contains zero values the result is not defined
     240      var original2 = new double[] { 1, 2, 0, 0, 0 };
     241      OnlineMeanAbsolutePercentageErrorCalculator.Calculate(original2, estimated, out errorState);
     242      Assert.AreEqual(OnlineCalculatorError.InvalidValueAdded, errorState);
     243    }
    142244  }
    143245}
Note: See TracChangeset for help on using the changeset viewer.