Free cookie consent management tool by TermsFeed Policy Generator

Changeset 6974


Ignore:
Timestamp:
11/09/11 10:59:31 (13 years ago)
Author:
gkronber
Message:

#1081: cleared up definition of accuracy metrics for time series prognosis to make the distinction between one n-step forecast and n one-step forecasts clearer. Implemented calculators for time series accuracy metrics to support the calculation of the average accuracy over m n-step forecasts and adapted the unit tests accordingly.

Location:
trunk/sources
Files:
1 added
6 edited

Legend:

Unmodified
Added
Removed

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/HeuristicLab.Problems.DataAnalysis-3.4.csproj

    r6913 r6974  
    143143    <Compile Include="Interfaces\Regression\IRegressionEnsembleSolution.cs" />
    144144    <Compile Include="Implementation\Regression\RegressionSolutionBase.cs" />
     145    <Compile Include="Interfaces\TimeSeriesPrognosis\IOnlineTimeSeriesCalculator.cs" />
    145146    <Compile Include="Interfaces\TimeSeriesPrognosis\ITimeSeriesPrognosisModel.cs" />
    146147    <Compile Include="Interfaces\TimeSeriesPrognosis\ITimeSeriesPrognosisProblem.cs" />

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisSolutionBase.cs

    r6961 r6974  
    4040    private const string TrainingNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (training)";
    4141    private const string TestNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (test)";
    42     private const string TrainingDirectionalSymmetryResultName = "Directional symmetry (training)";
    43     private const string TestDirectionalSymmetryResultName = "Directional symmetry (test)";
    44     private const string TrainingWeightedDirectionalSymmetryResultName = "Weighted directional symmetry (training)";
    45     private const string TestWeightedDirectionalSymmetryResultName = "Weighted directional symmetry (test)";
    46     private const string TrainingTheilsUStatisticResultName = "Theil's U (training)";
    47     private const string TestTheilsUStatisticResultName = "Theil's U (test)";
     42    private const string TrainingDirectionalSymmetryResultName = "Average directional symmetry (training)";
     43    private const string TestDirectionalSymmetryResultName = "Average directional symmetry (test)";
     44    private const string TrainingWeightedDirectionalSymmetryResultName = "Average weighted directional symmetry (training)";
     45    private const string TestWeightedDirectionalSymmetryResultName = "Average weighted directional symmetry (test)";
     46    private const string TrainingTheilsUStatisticResultName = "Average Theil's U (training)";
     47    private const string TestTheilsUStatisticResultName = "Average Theil's U (test)";
    4848
    4949    public new ITimeSeriesPrognosisModel Model {
    … …  
    146146      Add(new Result(TrainingNormalizedMeanSquaredErrorResultName, "Normalized mean of squared errors of the model on the training partition", new DoubleValue()));
    147147      Add(new Result(TestNormalizedMeanSquaredErrorResultName, "Normalized mean of squared errors of the model on the test partition", new DoubleValue()));
    148       Add(new Result(TrainingDirectionalSymmetryResultName, "The directional symmetry of the output of the model on the training partition", new DoubleValue()));
    149       Add(new Result(TestDirectionalSymmetryResultName, "The directional symmetry of the output of the model on the test partition", new DoubleValue()));
    150       Add(new Result(TrainingWeightedDirectionalSymmetryResultName, "The weighted directional symmetry of the output of the model on the training partition", new DoubleValue()));
    151       Add(new Result(TestWeightedDirectionalSymmetryResultName, "The weighted directional symmetry of the output of the model on the test partition", new DoubleValue()));
    152       Add(new Result(TrainingTheilsUStatisticResultName, "The Theil's U statistic of the output of the model on the training partition", new DoubleValue()));
    153       Add(new Result(TestTheilsUStatisticResultName, "The Theil's U statistic of the output of the model on the test partition", new DoubleValue()));
     148      Add(new Result(TrainingDirectionalSymmetryResultName, "The average directional symmetry of the forecasts of the model on the training partition", new PercentValue()));
     149      Add(new Result(TestDirectionalSymmetryResultName, "The average directional symmetry of the forecasts of the model on the test partition", new PercentValue()));
     150      Add(new Result(TrainingWeightedDirectionalSymmetryResultName, "The average weighted directional symmetry of the forecasts of the model on the training partition", new DoubleValue()));
     151      Add(new Result(TestWeightedDirectionalSymmetryResultName, "The average weighted directional symmetry of the forecasts of the model on the test partition", new DoubleValue()));
     152      Add(new Result(TrainingTheilsUStatisticResultName, "The average Theil's U statistic of the forecasts of the model on the training partition", new DoubleValue()));
     153      Add(new Result(TestTheilsUStatisticResultName, "The average Theil's U statistic of the forecasts of the model on the test partition", new DoubleValue()));
    154154    }
    155155
    … …  
    191191      TestNormalizedMeanSquaredError = errorState == OnlineCalculatorError.None ? testNmse : double.NaN;
    192192
    193       double trainingDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
     193      var startTrainingValues = originalTrainingValues;
     194      // each continuation is only one element long
     195      var actualContinuationsTraining = from x in originalTrainingValues.Skip(1)
     196                                        select Enumerable.Repeat(x, 1);
     197      // each forecast is only one elemnt long
     198      // disregards the first estimated value (we could include this again by extending the list of original values by one step to the left
     199      // this is the easier way
     200      var predictedContinuationsTraining = from x in estimatedTrainingValues.Skip(1)
     201                                           select Enumerable.Repeat(x, 1);
     202
     203      var startTestValues = originalTestValues;
     204      var actualContinuationsTest = from x in originalTestValues.Skip(1)
     205                                    select Enumerable.Repeat(x, 1);
     206      var predictedContinuationsTest = from x in estimatedTestValues.Skip(1)
     207                                       select Enumerable.Repeat(x, 1);
     208
     209      double trainingDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(startTrainingValues, actualContinuationsTraining, predictedContinuationsTraining, out errorState);
    194210      TrainingDirectionalSymmetry = errorState == OnlineCalculatorError.None ? trainingDirectionalSymmetry : double.NaN;
    195       double testDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
     211      double testDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(startTestValues, actualContinuationsTest, predictedContinuationsTest, out errorState);
    196212      TestDirectionalSymmetry = errorState == OnlineCalculatorError.None ? testDirectionalSymmetry : double.NaN;
    197213
    198       double trainingWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
     214      double trainingWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(startTrainingValues, actualContinuationsTraining, predictedContinuationsTraining, out errorState);
    199215      TrainingWeightedDirectionalSymmetry = errorState == OnlineCalculatorError.None ? trainingWeightedDirectionalSymmetry : double.NaN;
    200       double testWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
     216      double testWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(startTestValues, actualContinuationsTest, predictedContinuationsTest, out errorState);
    201217      TestWeightedDirectionalSymmetry = errorState == OnlineCalculatorError.None ? testWeightedDirectionalSymmetry : double.NaN;
    202218
    203       double trainingTheilsU = OnlineTheilsUStatisticCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
     219      double trainingTheilsU = OnlineTheilsUStatisticCalculator.Calculate(startTrainingValues, actualContinuationsTraining, predictedContinuationsTraining, out errorState);
    204220      TrainingTheilsUStatistic = errorState == OnlineCalculatorError.None ? trainingTheilsU : double.NaN;
    205       double testTheilsU = OnlineTheilsUStatisticCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
     221      double testTheilsU = OnlineTheilsUStatisticCalculator.Calculate(startTestValues, actualContinuationsTest, predictedContinuationsTest, out errorState);
    206222      TestTheilsUStatistic = errorState == OnlineCalculatorError.None ? testTheilsU : double.NaN;
    207 
    208 
    209223    }
    210224  }

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/OnlineDirectionalSymmetryCalculator.cs

    r6964 r6974  
    2222using System;
    2323using System.Collections.Generic;
     24using HeuristicLab.Common;
    2425
    2526
    2627namespace HeuristicLab.Problems.DataAnalysis {
    27   public class OnlineDirectionalSymmetryCalculator : IOnlineCalculator {
    28     private double prevEstimated;
    29     private double prevOriginal;
     28  public class OnlineDirectionalSymmetryCalculator : IOnlineTimeSeriesCalculator {
    3029    private int n;
    3130    private int nCorrect;
    … …  
    3332    public double DirectionalSymmetry {
    3433      get {
    35         if (n <= 1) return 0.0;
    36         return (double)nCorrect / (n - 1) * 100.0;
     34        if (n < 1) return 0.0;
     35        return (double)nCorrect / n;
    3736      }
    3837    }
    … …  
    5150    }
    5251
    53     public void Add(double original, double estimated) {
    54       if (double.IsNaN(estimated) || double.IsInfinity(estimated) || double.IsNaN(original) || double.IsInfinity(original) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
     52    public void Add(double startValue, IEnumerable<double> actualContinuation, IEnumerable<double> predictedContinuation) {
     53      if (double.IsNaN(startValue) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
    5554        errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    56       } else if (n == 0) {
    57         prevOriginal = original;
    58         prevEstimated = estimated;
    59         n++;
    6055      } else {
    61         if ((original - prevOriginal) * (estimated - prevEstimated) >= 0.0) {
    62           nCorrect++;
     56        var actualEnumerator = actualContinuation.GetEnumerator();
     57        var predictedEnumerator = predictedContinuation.GetEnumerator();
     58        double prevActual = startValue;
     59        double prevPredicted = startValue;
     60        while (actualEnumerator.MoveNext() & predictedEnumerator.MoveNext() & errorState != OnlineCalculatorError.InvalidValueAdded) {
     61          double actual = actualEnumerator.Current;
     62          double predicted = predictedEnumerator.Current;
     63          if (double.IsNaN(actual) || double.IsNaN(predicted)) {
     64            errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     65          } else {
     66            // count a prediction correct if the trend (positive/negative/no change) is predicted correctly
     67            if ((actual - prevActual) * (predicted - prevPredicted) > 0.0 ||
     68                (actual - prevActual).IsAlmost(predicted - prevPredicted)
     69              ) {
     70              nCorrect++;
     71            }
     72            n++;
     73          }
    6374        }
    64         n++;
    65         errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded);        // n >= 1
    66         prevOriginal = original;
    67         prevEstimated = estimated;
     75        // check if both enumerators are at the end to make sure both enumerations have the same length
     76        if (actualEnumerator.MoveNext() || predictedEnumerator.MoveNext()) {
     77          errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     78        } else {
     79          errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded); // n >= 1
     80        }
    6881      }
    6982    }
    … …  
    7285      n = 0;
    7386      nCorrect = 0;
    74       prevOriginal = double.NaN;
    75       prevEstimated = double.NaN;
    7687      errorState = OnlineCalculatorError.InsufficientElementsAdded;
    7788    }
    7889
    7990
    80     public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
    81       IEnumerator<double> originalEnumerator = originalValues.GetEnumerator();
    82       IEnumerator<double> estimatedEnumerator = estimatedValues.GetEnumerator();
     91    public static double Calculate(IEnumerable<double> startValues, IEnumerable<IEnumerable<double>> actualContinuations, IEnumerable<IEnumerable<double>> predictedContinuations, out OnlineCalculatorError errorState) {
     92      IEnumerator<double> startValueEnumerator = startValues.GetEnumerator();
     93      IEnumerator<IEnumerable<double>> actualContinuationsEnumerator = actualContinuations.GetEnumerator();
     94      IEnumerator<IEnumerable<double>> predictedContinuationsEnumerator = predictedContinuations.GetEnumerator();
    8395      OnlineDirectionalSymmetryCalculator dsCalculator = new OnlineDirectionalSymmetryCalculator();
    8496
    85       // add first element of time series as a reference point
    86       originalEnumerator.MoveNext();
    87       estimatedEnumerator.MoveNext();
    88       dsCalculator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
    89      
    90       // always move forward both enumerators (do not use short-circuit evaluation!)
    91       while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
    92         double original = originalEnumerator.Current;
    93         double estimated = estimatedEnumerator.Current;
    94         dsCalculator.Add(original, estimated);
     97      // always move forward all enumerators (do not use short-circuit evaluation!)
     98      while (startValueEnumerator.MoveNext() & actualContinuationsEnumerator.MoveNext() & predictedContinuationsEnumerator.MoveNext()) {
     99        dsCalculator.Add(startValueEnumerator.Current, actualContinuationsEnumerator.Current, predictedContinuationsEnumerator.Current);
    95100        if (dsCalculator.ErrorState != OnlineCalculatorError.None) break;
    96101      }
    97102
    98       // check if both enumerators are at the end to make sure both enumerations have the same length
     103      // check if all enumerators are at the end to make sure both enumerations have the same length
    99104      if (dsCalculator.ErrorState == OnlineCalculatorError.None &&
    100           (originalEnumerator.MoveNext() || estimatedEnumerator.MoveNext())) {
    101         throw new ArgumentException("Number of elements in originalValues and estimatedValues enumerations doesn't match.");
     105          (startValueEnumerator.MoveNext() || actualContinuationsEnumerator.MoveNext() || predictedContinuationsEnumerator.MoveNext())) {
     106        throw new ArgumentException("Number of elements in startValues, actualContinuations and estimatedValues predictedContinuations doesn't match.");
    102107      } else {
    103108        errorState = dsCalculator.ErrorState;

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/OnlineTheilsUStatisticCalculator.cs

    r6964 r6974  
    2222using System;
    2323using System.Collections.Generic;
     24using HeuristicLab.Common;
    2425
    2526namespace HeuristicLab.Problems.DataAnalysis {
    26   public class OnlineTheilsUStatisticCalculator : IOnlineCalculator {
     27  public class OnlineTheilsUStatisticCalculator : IOnlineTimeSeriesCalculator {
    2728    private OnlineMeanAndVarianceCalculator squaredErrorMeanCalculator;
    2829    private OnlineMeanAndVarianceCalculator unbiasedEstimatorMeanCalculator;
    29     private double prevOriginal;
    30     private int n;
    3130
    3231    public double TheilsUStatistic {
    … …  
    5251    }
    5352
    54     public void Add(double original, double estimated) {
    55       if (double.IsNaN(estimated) || double.IsInfinity(estimated) || double.IsNaN(original) || double.IsInfinity(original) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
     53    public void Add(double startValue, IEnumerable<double> actualContinuation, IEnumerable<double> predictedContinuation) {
     54      if (double.IsNaN(startValue) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
    5655        errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    57       } else if (n == 0) {
    58         prevOriginal = original;
    59         n++;
    6056      } else {
    61         // error of predicted change
    62         double errorEstimatedChange = (estimated - original);
    63         squaredErrorMeanCalculator.Add(errorEstimatedChange * errorEstimatedChange);
     57        var actualEnumerator = actualContinuation.GetEnumerator();
     58        var predictedEnumerator = predictedContinuation.GetEnumerator();
     59        while (actualEnumerator.MoveNext() & predictedEnumerator.MoveNext() & ErrorState != OnlineCalculatorError.InvalidValueAdded) {
     60          double actual = actualEnumerator.Current;
     61          double predicted = predictedEnumerator.Current;
     62          if (double.IsNaN(actual) || double.IsNaN(predicted)) {
     63            errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     64          } else {
     65            // error of predicted change
     66            double errorPredictedChange = (predicted - startValue) - (actual - startValue);
     67            squaredErrorMeanCalculator.Add(errorPredictedChange * errorPredictedChange);
    6468
    65         double errorNoChange = (original - prevOriginal);
    66         unbiasedEstimatorMeanCalculator.Add(errorNoChange * errorNoChange);
    67         errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded);        // n >= 1
    68         prevOriginal = original;
    69         n++;
     69            double errorNoChange = (actual - startValue);
     70            unbiasedEstimatorMeanCalculator.Add(errorNoChange * errorNoChange);
     71          }
     72        }
     73        // check if both enumerators are at the end to make sure both enumerations have the same length
     74        if (actualEnumerator.MoveNext() || predictedEnumerator.MoveNext()) {
     75          errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     76        } else {
     77          errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded); // n >= 1
     78        }
    7079      }
    7180    }
    7281
    73 
    7482    public void Reset() {
    75       prevOriginal = double.NaN;
    76       n = 0;
    7783      squaredErrorMeanCalculator.Reset();
    7884      unbiasedEstimatorMeanCalculator.Reset();
    … …  
    8288    #endregion
    8389
    84     public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
    85       IEnumerator<double> originalValuesEnumerator = originalValues.GetEnumerator();
    86       IEnumerator<double> estimatedValuesEnumerator = estimatedValues.GetEnumerator();
     90    public static double Calculate(IEnumerable<double> startValues, IEnumerable<IEnumerable<double>> actualContinuations, IEnumerable<IEnumerable<double>> predictedContinuations, out OnlineCalculatorError errorState) {
     91      IEnumerator<double> startValueEnumerator = startValues.GetEnumerator();
     92      IEnumerator<IEnumerable<double>> actualContinuationsEnumerator = actualContinuations.GetEnumerator();
     93      IEnumerator<IEnumerable<double>> predictedContinuationsEnumerator = predictedContinuations.GetEnumerator();
    8794      OnlineTheilsUStatisticCalculator calculator = new OnlineTheilsUStatisticCalculator();
    8895
    89       // add first element of time series as a reference point
    90       originalValuesEnumerator.MoveNext();
    91       estimatedValuesEnumerator.MoveNext();
    92       calculator.Add(originalValuesEnumerator.Current, estimatedValuesEnumerator.Current);
    93 
    94       // always move forward both enumerators (do not use short-circuit evaluation!)
    95       while (originalValuesEnumerator.MoveNext() & estimatedValuesEnumerator.MoveNext()) {
    96         double estimated = estimatedValuesEnumerator.Current;
    97         double original = originalValuesEnumerator.Current;
    98         calculator.Add(original, estimated);
     96      // always move forward all enumerators (do not use short-circuit evaluation!)
     97      while (startValueEnumerator.MoveNext() & actualContinuationsEnumerator.MoveNext() & predictedContinuationsEnumerator.MoveNext()) {
     98        calculator.Add(startValueEnumerator.Current, actualContinuationsEnumerator.Current, predictedContinuationsEnumerator.Current);
    9999        if (calculator.ErrorState != OnlineCalculatorError.None) break;
    100100      }
    101101
    102       // check if both enumerators are at the end to make sure both enumerations have the same length
     102      // check if all enumerators are at the end to make sure both enumerations have the same length
    103103      if (calculator.ErrorState == OnlineCalculatorError.None &&
    104           (estimatedValuesEnumerator.MoveNext() || originalValuesEnumerator.MoveNext())) {
    105         throw new ArgumentException("Number of elements in originalValues and estimatedValues enumerations doesn't match.");
     104          (startValueEnumerator.MoveNext() || actualContinuationsEnumerator.MoveNext() || predictedContinuationsEnumerator.MoveNext())) {
     105        throw new ArgumentException("Number of elements in startValues, actualContinuations and estimatedValues predictedContinuations doesn't match.");
    106106      } else {
    107107        errorState = calculator.ErrorState;

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/OnlineWeightedDirectionalSymmetryCalculator.cs

    r6964 r6974  
    2222using System;
    2323using System.Collections.Generic;
     24using HeuristicLab.Common;
    2425
    2526
    2627namespace HeuristicLab.Problems.DataAnalysis {
    27   public class OnlineWeightedDirectionalSymmetryCalculator : IOnlineCalculator {
     28  public class OnlineWeightedDirectionalSymmetryCalculator : IOnlineTimeSeriesCalculator {
    2829    private double prevEstimated;
    2930    private double prevOriginal;
    … …  
    5253    }
    5354
    54     public void Add(double original, double estimated) {
    55       if (double.IsNaN(estimated) || double.IsInfinity(estimated) || double.IsNaN(original) || double.IsInfinity(original) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
     55    public void Add(double startValue, IEnumerable<double> actualContinuation, IEnumerable<double> predictedContinuation) {
     56      if (double.IsNaN(startValue) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
    5657        errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    57       } else if (n == 0) {
    58         prevOriginal = original;
    59         prevEstimated = estimated;
    60         n++;
    6158      } else {
    62         double err = Math.Abs(original - estimated);
    63         if ((original - prevOriginal) * (estimated - prevEstimated) >= 0.0) {
    64           correctSum += err;
     59        var actualEnumerator = actualContinuation.GetEnumerator();
     60        var predictedEnumerator = predictedContinuation.GetEnumerator();
     61        while (actualEnumerator.MoveNext() & predictedEnumerator.MoveNext() & errorState != OnlineCalculatorError.InvalidValueAdded) {
     62          double actual = actualEnumerator.Current;
     63          double predicted = predictedEnumerator.Current;
     64          if (double.IsNaN(actual) || double.IsNaN(predicted)) {
     65            errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     66          } else {
     67            double err = Math.Abs(actual - predicted);
     68            // count as correct only if the trend (positive/negative/no change) is predicted correctly
     69            if ((actual - startValue) * (predicted - startValue) > 0.0 ||
     70              (actual - startValue).IsAlmost(predicted - startValue)) {
     71              correctSum += err;
     72            } else {
     73              incorrectSum += err;
     74            }
     75            n++;
     76          }
     77        }
     78        // check if both enumerators are at the end to make sure both enumerations have the same length
     79        if (actualEnumerator.MoveNext() || predictedEnumerator.MoveNext()) {
     80          errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    6581        } else {
    66           incorrectSum += err;
     82          errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded); // n >= 1
    6783        }
    68         n++;
    69         errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded);        // n >= 1
    70         prevOriginal = original;
    71         prevEstimated = estimated;
    7284      }
    7385    }
    … …  
    8395
    8496
    85     public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
    86       IEnumerator<double> originalEnumerator = originalValues.GetEnumerator();
    87       IEnumerator<double> estimatedEnumerator = estimatedValues.GetEnumerator();
    88       OnlineWeightedDirectionalSymmetryCalculator dsCalculator = new OnlineWeightedDirectionalSymmetryCalculator();
     97    public static double Calculate(IEnumerable<double> startValues, IEnumerable<IEnumerable<double>> actualContinuations, IEnumerable<IEnumerable<double>> predictedContinuations, out OnlineCalculatorError errorState) {
     98      IEnumerator<double> startValueEnumerator = startValues.GetEnumerator();
     99      IEnumerator<IEnumerable<double>> actualContinuationsEnumerator = actualContinuations.GetEnumerator();
     100      IEnumerator<IEnumerable<double>> predictedContinuationsEnumerator = predictedContinuations.GetEnumerator();
     101      OnlineWeightedDirectionalSymmetryCalculator calculator = new OnlineWeightedDirectionalSymmetryCalculator();
    89102
    90       // add first element of time series as a reference point
    91       originalEnumerator.MoveNext();
    92       estimatedEnumerator.MoveNext();
    93       dsCalculator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
    94 
    95       // always move forward both enumerators (do not use short-circuit evaluation!)
    96       while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
    97         double original = originalEnumerator.Current;
    98         double estimated = estimatedEnumerator.Current;
    99         dsCalculator.Add(original, estimated);
    100         if (dsCalculator.ErrorState != OnlineCalculatorError.None) break;
     103      // always move forward all enumerators (do not use short-circuit evaluation!)
     104      while (startValueEnumerator.MoveNext() & actualContinuationsEnumerator.MoveNext() & predictedContinuationsEnumerator.MoveNext()) {
     105        calculator.Add(startValueEnumerator.Current, actualContinuationsEnumerator.Current, predictedContinuationsEnumerator.Current);
     106        if (calculator.ErrorState != OnlineCalculatorError.None) break;
    101107      }
    102108
    103       // check if both enumerators are at the end to make sure both enumerations have the same length
    104       if (dsCalculator.ErrorState == OnlineCalculatorError.None &&
    105           (originalEnumerator.MoveNext() || estimatedEnumerator.MoveNext())) {
    106         throw new ArgumentException("Number of elements in originalValues and estimatedValues enumerations doesn't match.");
     109      // check if all enumerators are at the end to make sure both enumerations have the same length
     110      if (calculator.ErrorState == OnlineCalculatorError.None &&
     111          (startValueEnumerator.MoveNext() || actualContinuationsEnumerator.MoveNext() || predictedContinuationsEnumerator.MoveNext())) {
     112        throw new ArgumentException("Number of elements in startValues, actualContinuations and estimatedValues predictedContinuations doesn't match.");
    107113      } else {
    108         errorState = dsCalculator.ErrorState;
    109         return dsCalculator.WeightedDirectionalSymmetry;
     114        errorState = calculator.ErrorState;
     115        return calculator.WeightedDirectionalSymmetry;
    110116      }
    111117    }

  • trunk/sources/HeuristicLab.Tests/HeuristicLab.Problems.DataAnalysis-3.4/StatisticCalculatorsTest.cs

    r6880 r6974  
    2020#endregion
    2121
     22using System;
    2223using System.Collections.Generic;
    2324using System.Linq;
    … …  
    140141      }
    141142    }
     143
     144    [TestMethod]
     145    public void CalculateDirectionalSymmetryTest() {
     146      // delta: +0.01, +1, -0.01, -2, -0.01, -1, +0.01, +2
     147      var original = new double[]
     148                       {
     149                         0,
     150                         0.01,
     151                         1.01,
     152                         1,
     153                         -1,
     154                         -1.01,
     155                         -2.01,
     156                         -2,
     157                         0
     158                       };
     159      // delta to original(t-1): +1, +0, -1, -0, -1, +0.01, +0.01, +2
     160      var estimated = new double[]
     161                        {
     162                          -1,
     163                          1,
     164                          0.01,
     165                          0.01,
     166                          1,
     167                          -1,
     168                          -1.02,
     169                          -2.02,
     170                          0
     171                        };
     172
     173      // one-step forecast
     174      var startValues = original;
     175      var actualContinuations = from x in original.Skip(1)
     176                                select Enumerable.Repeat(x, 1);
     177      var predictedContinuations = from x in estimated.Skip(1)
     178                                   select Enumerable.Repeat(x, 1);
     179      double expected = 0.5;  // half of the predicted deltas are correct
     180      OnlineCalculatorError errorState;
     181      double actual = OnlineDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
     182      Assert.AreEqual(expected, actual, 1E-9);
     183    }
     184    [TestMethod]
     185    public void CalculateWeightedDirectionalSymmetryTest() {
     186      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 }; // +0.01, +1, -0.01, -2, -0.01, -1, +0.01, +2
     187      var estimated = new double[] { 1, 2, 2, 1, 1, 0, 0.01, 0.02, 2.02 }; // delta to original: +2, +1.99, -0.01, 0, +1, -1.02, +2.01, +4.02
     188      // one-step forecast
     189      var startValues = original;
     190      var actualContinuations = from x in original.Skip(1)
     191                                select Enumerable.Repeat(x, 1);
     192      var predictedContinuations = from x in estimated.Skip(1)
     193                                   select Enumerable.Repeat(x, 1);
     194      // absolute errors = 1.99, 0.99, 0, 2, 1.01, 2.02, 2.02, 2.02     
     195      // sum of absolute errors for correctly predicted deltas = 2.97
     196      // sum of absolute errors for incorrectly predicted deltas = 3.03
     197      double expected = 5.03 / 7.02;
     198      OnlineCalculatorError errorState;
     199      double actual = OnlineWeightedDirectionalSymmetryCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
     200      Assert.AreEqual(expected, actual, 1E-9);
     201    }
     202    [TestMethod]
     203    public void CalculateTheilsUTest() {
     204      var original = new double[] { 0, 0.01, 1.01, 1, -1, -1.01, -2.01, -2, 0 };
     205      var estimated = new double[] { 1, 1.01, 0.01, 2, 0, -0.01, -1.01, -3, 1 };
     206      // one-step forecast
     207      var startValues = original;
     208      var actualContinuations = from x in original.Skip(1)
     209                                select Enumerable.Repeat(x, 1);
     210      var predictedContinuations = from x in estimated.Skip(1)
     211                                   select Enumerable.Repeat(x, 1);
     212      // Sum of squared errors of model y(t+1) = y(t) = 10.0004
     213      // Sum of squared errors of predicted values = 8 
     214      double expected = Math.Sqrt(8 / 10.0004);
     215      OnlineCalculatorError errorState;
     216      double actual = OnlineTheilsUStatisticCalculator.Calculate(startValues, actualContinuations, predictedContinuations, out errorState);
     217      Assert.AreEqual(expected, actual, 1E-9);
     218    }
     219    [TestMethod]
     220    public void CalculateAccuracyTest() {
     221      var original = new double[] { 1, 1, 0, 0 };
     222      var estimated = new double[] { 1, 0, 1, 0 };
     223      double expected = 0.5;
     224      OnlineCalculatorError errorState;
     225      double actual = OnlineAccuracyCalculator.Calculate(original, estimated, out errorState);
     226      Assert.AreEqual(expected, actual, 1E-9);
     227    }
     228
     229    [TestMethod]
     230    public void CalculateMeanAbsolutePercentageErrorTest() {
     231      var original = new double[] { 1, 2, 3, 1, 5 };
     232      var estimated = new double[] { 2, 1, 3, 1, 0 };
     233      double expected = 0.5;
     234      OnlineCalculatorError errorState;
     235      double actual = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(original, estimated, out errorState);
     236      Assert.AreEqual(expected, actual, 1E-9);
     237      Assert.AreEqual(OnlineCalculatorError.None, errorState);
     238
     239      // if the original contains zero values the result is not defined
     240      var original2 = new double[] { 1, 2, 0, 0, 0 };
     241      OnlineMeanAbsolutePercentageErrorCalculator.Calculate(original2, estimated, out errorState);
     242      Assert.AreEqual(OnlineCalculatorError.InvalidValueAdded, errorState);
     243    }
    142244  }
    143245}
Note: See TracChangeset for help on using the changeset viewer.