Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
11/09/11 10:59:31 (13 years ago)
Author:
gkronber
Message:

#1081: cleared up definition of accuracy metrics for time series prognosis to make the distinction between one n-step forecast and n one-step forecasts clearer. Implemented calculators for time series accuracy metrics to support the calculation of the average accuracy over m n-step forecasts and adapted the unit tests accordingly.

Location:
trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/OnlineDirectionalSymmetryCalculator.cs

    r6964 r6974  
    2222using System;
    2323using System.Collections.Generic;
     24using HeuristicLab.Common;
    2425
    2526
    2627namespace HeuristicLab.Problems.DataAnalysis {
    27   public class OnlineDirectionalSymmetryCalculator : IOnlineCalculator {
    28     private double prevEstimated;
    29     private double prevOriginal;
     28  public class OnlineDirectionalSymmetryCalculator : IOnlineTimeSeriesCalculator {
    3029    private int n;
    3130    private int nCorrect;
    … …  
    3332    public double DirectionalSymmetry {
    3433      get {
    35         if (n <= 1) return 0.0;
    36         return (double)nCorrect / (n - 1) * 100.0;
     34        if (n < 1) return 0.0;
     35        return (double)nCorrect / n;
    3736      }
    3837    }
    … …  
    5150    }
    5251
    53     public void Add(double original, double estimated) {
    54       if (double.IsNaN(estimated) || double.IsInfinity(estimated) || double.IsNaN(original) || double.IsInfinity(original) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
     52    public void Add(double startValue, IEnumerable<double> actualContinuation, IEnumerable<double> predictedContinuation) {
     53      if (double.IsNaN(startValue) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
    5554        errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    56       } else if (n == 0) {
    57         prevOriginal = original;
    58         prevEstimated = estimated;
    59         n++;
    6055      } else {
    61         if ((original - prevOriginal) * (estimated - prevEstimated) >= 0.0) {
    62           nCorrect++;
     56        var actualEnumerator = actualContinuation.GetEnumerator();
     57        var predictedEnumerator = predictedContinuation.GetEnumerator();
     58        double prevActual = startValue;
     59        double prevPredicted = startValue;
     60        while (actualEnumerator.MoveNext() & predictedEnumerator.MoveNext() & errorState != OnlineCalculatorError.InvalidValueAdded) {
     61          double actual = actualEnumerator.Current;
     62          double predicted = predictedEnumerator.Current;
     63          if (double.IsNaN(actual) || double.IsNaN(predicted)) {
     64            errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     65          } else {
     66            // count a prediction correct if the trend (positive/negative/no change) is predicted correctly
     67            if ((actual - prevActual) * (predicted - prevPredicted) > 0.0 ||
     68                (actual - prevActual).IsAlmost(predicted - prevPredicted)
     69              ) {
     70              nCorrect++;
     71            }
     72            n++;
     73          }
    6374        }
    64         n++;
    65         errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded);        // n >= 1
    66         prevOriginal = original;
    67         prevEstimated = estimated;
     75        // check if both enumerators are at the end to make sure both enumerations have the same length
     76        if (actualEnumerator.MoveNext() || predictedEnumerator.MoveNext()) {
     77          errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     78        } else {
     79          errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded); // n >= 1
     80        }
    6881      }
    6982    }
    … …  
    7285      n = 0;
    7386      nCorrect = 0;
    74       prevOriginal = double.NaN;
    75       prevEstimated = double.NaN;
    7687      errorState = OnlineCalculatorError.InsufficientElementsAdded;
    7788    }
    7889
    7990
    80     public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
    81       IEnumerator<double> originalEnumerator = originalValues.GetEnumerator();
    82       IEnumerator<double> estimatedEnumerator = estimatedValues.GetEnumerator();
     91    public static double Calculate(IEnumerable<double> startValues, IEnumerable<IEnumerable<double>> actualContinuations, IEnumerable<IEnumerable<double>> predictedContinuations, out OnlineCalculatorError errorState) {
     92      IEnumerator<double> startValueEnumerator = startValues.GetEnumerator();
     93      IEnumerator<IEnumerable<double>> actualContinuationsEnumerator = actualContinuations.GetEnumerator();
     94      IEnumerator<IEnumerable<double>> predictedContinuationsEnumerator = predictedContinuations.GetEnumerator();
    8395      OnlineDirectionalSymmetryCalculator dsCalculator = new OnlineDirectionalSymmetryCalculator();
    8496
    85       // add first element of time series as a reference point
    86       originalEnumerator.MoveNext();
    87       estimatedEnumerator.MoveNext();
    88       dsCalculator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
    89      
    90       // always move forward both enumerators (do not use short-circuit evaluation!)
    91       while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
    92         double original = originalEnumerator.Current;
    93         double estimated = estimatedEnumerator.Current;
    94         dsCalculator.Add(original, estimated);
     97      // always move forward all enumerators (do not use short-circuit evaluation!)
     98      while (startValueEnumerator.MoveNext() & actualContinuationsEnumerator.MoveNext() & predictedContinuationsEnumerator.MoveNext()) {
     99        dsCalculator.Add(startValueEnumerator.Current, actualContinuationsEnumerator.Current, predictedContinuationsEnumerator.Current);
    95100        if (dsCalculator.ErrorState != OnlineCalculatorError.None) break;
    96101      }
    97102
    98       // check if both enumerators are at the end to make sure both enumerations have the same length
     103      // check if all enumerators are at the end to make sure both enumerations have the same length
    99104      if (dsCalculator.ErrorState == OnlineCalculatorError.None &&
    100           (originalEnumerator.MoveNext() || estimatedEnumerator.MoveNext())) {
    101         throw new ArgumentException("Number of elements in originalValues and estimatedValues enumerations doesn't match.");
     105          (startValueEnumerator.MoveNext() || actualContinuationsEnumerator.MoveNext() || predictedContinuationsEnumerator.MoveNext())) {
     106        throw new ArgumentException("Number of elements in startValues, actualContinuations and estimatedValues predictedContinuations doesn't match.");
    102107      } else {
    103108        errorState = dsCalculator.ErrorState;

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/OnlineTheilsUStatisticCalculator.cs

    r6964 r6974  
    2222using System;
    2323using System.Collections.Generic;
     24using HeuristicLab.Common;
    2425
    2526namespace HeuristicLab.Problems.DataAnalysis {
    26   public class OnlineTheilsUStatisticCalculator : IOnlineCalculator {
     27  public class OnlineTheilsUStatisticCalculator : IOnlineTimeSeriesCalculator {
    2728    private OnlineMeanAndVarianceCalculator squaredErrorMeanCalculator;
    2829    private OnlineMeanAndVarianceCalculator unbiasedEstimatorMeanCalculator;
    29     private double prevOriginal;
    30     private int n;
    3130
    3231    public double TheilsUStatistic {
    … …  
    5251    }
    5352
    54     public void Add(double original, double estimated) {
    55       if (double.IsNaN(estimated) || double.IsInfinity(estimated) || double.IsNaN(original) || double.IsInfinity(original) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
     53    public void Add(double startValue, IEnumerable<double> actualContinuation, IEnumerable<double> predictedContinuation) {
     54      if (double.IsNaN(startValue) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
    5655        errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    57       } else if (n == 0) {
    58         prevOriginal = original;
    59         n++;
    6056      } else {
    61         // error of predicted change
    62         double errorEstimatedChange = (estimated - original);
    63         squaredErrorMeanCalculator.Add(errorEstimatedChange * errorEstimatedChange);
     57        var actualEnumerator = actualContinuation.GetEnumerator();
     58        var predictedEnumerator = predictedContinuation.GetEnumerator();
     59        while (actualEnumerator.MoveNext() & predictedEnumerator.MoveNext() & ErrorState != OnlineCalculatorError.InvalidValueAdded) {
     60          double actual = actualEnumerator.Current;
     61          double predicted = predictedEnumerator.Current;
     62          if (double.IsNaN(actual) || double.IsNaN(predicted)) {
     63            errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     64          } else {
     65            // error of predicted change
     66            double errorPredictedChange = (predicted - startValue) - (actual - startValue);
     67            squaredErrorMeanCalculator.Add(errorPredictedChange * errorPredictedChange);
    6468
    65         double errorNoChange = (original - prevOriginal);
    66         unbiasedEstimatorMeanCalculator.Add(errorNoChange * errorNoChange);
    67         errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded);        // n >= 1
    68         prevOriginal = original;
    69         n++;
     69            double errorNoChange = (actual - startValue);
     70            unbiasedEstimatorMeanCalculator.Add(errorNoChange * errorNoChange);
     71          }
     72        }
     73        // check if both enumerators are at the end to make sure both enumerations have the same length
     74        if (actualEnumerator.MoveNext() || predictedEnumerator.MoveNext()) {
     75          errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     76        } else {
     77          errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded); // n >= 1
     78        }
    7079      }
    7180    }
    7281
    73 
    7482    public void Reset() {
    75       prevOriginal = double.NaN;
    76       n = 0;
    7783      squaredErrorMeanCalculator.Reset();
    7884      unbiasedEstimatorMeanCalculator.Reset();
    … …  
    8288    #endregion
    8389
    84     public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
    85       IEnumerator<double> originalValuesEnumerator = originalValues.GetEnumerator();
    86       IEnumerator<double> estimatedValuesEnumerator = estimatedValues.GetEnumerator();
     90    public static double Calculate(IEnumerable<double> startValues, IEnumerable<IEnumerable<double>> actualContinuations, IEnumerable<IEnumerable<double>> predictedContinuations, out OnlineCalculatorError errorState) {
     91      IEnumerator<double> startValueEnumerator = startValues.GetEnumerator();
     92      IEnumerator<IEnumerable<double>> actualContinuationsEnumerator = actualContinuations.GetEnumerator();
     93      IEnumerator<IEnumerable<double>> predictedContinuationsEnumerator = predictedContinuations.GetEnumerator();
    8794      OnlineTheilsUStatisticCalculator calculator = new OnlineTheilsUStatisticCalculator();
    8895
    89       // add first element of time series as a reference point
    90       originalValuesEnumerator.MoveNext();
    91       estimatedValuesEnumerator.MoveNext();
    92       calculator.Add(originalValuesEnumerator.Current, estimatedValuesEnumerator.Current);
    93 
    94       // always move forward both enumerators (do not use short-circuit evaluation!)
    95       while (originalValuesEnumerator.MoveNext() & estimatedValuesEnumerator.MoveNext()) {
    96         double estimated = estimatedValuesEnumerator.Current;
    97         double original = originalValuesEnumerator.Current;
    98         calculator.Add(original, estimated);
     96      // always move forward all enumerators (do not use short-circuit evaluation!)
     97      while (startValueEnumerator.MoveNext() & actualContinuationsEnumerator.MoveNext() & predictedContinuationsEnumerator.MoveNext()) {
     98        calculator.Add(startValueEnumerator.Current, actualContinuationsEnumerator.Current, predictedContinuationsEnumerator.Current);
    9999        if (calculator.ErrorState != OnlineCalculatorError.None) break;
    100100      }
    101101
    102       // check if both enumerators are at the end to make sure both enumerations have the same length
     102      // check if all enumerators are at the end to make sure both enumerations have the same length
    103103      if (calculator.ErrorState == OnlineCalculatorError.None &&
    104           (estimatedValuesEnumerator.MoveNext() || originalValuesEnumerator.MoveNext())) {
    105         throw new ArgumentException("Number of elements in originalValues and estimatedValues enumerations doesn't match.");
     104          (startValueEnumerator.MoveNext() || actualContinuationsEnumerator.MoveNext() || predictedContinuationsEnumerator.MoveNext())) {
     105        throw new ArgumentException("Number of elements in startValues, actualContinuations and estimatedValues predictedContinuations doesn't match.");
    106106      } else {
    107107        errorState = calculator.ErrorState;

  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/OnlineCalculators/OnlineWeightedDirectionalSymmetryCalculator.cs

    r6964 r6974  
    2222using System;
    2323using System.Collections.Generic;
     24using HeuristicLab.Common;
    2425
    2526
    2627namespace HeuristicLab.Problems.DataAnalysis {
    27   public class OnlineWeightedDirectionalSymmetryCalculator : IOnlineCalculator {
     28  public class OnlineWeightedDirectionalSymmetryCalculator : IOnlineTimeSeriesCalculator {
    2829    private double prevEstimated;
    2930    private double prevOriginal;
    … …  
    5253    }
    5354
    54     public void Add(double original, double estimated) {
    55       if (double.IsNaN(estimated) || double.IsInfinity(estimated) || double.IsNaN(original) || double.IsInfinity(original) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
     55    public void Add(double startValue, IEnumerable<double> actualContinuation, IEnumerable<double> predictedContinuation) {
     56      if (double.IsNaN(startValue) || (errorState & OnlineCalculatorError.InvalidValueAdded) > 0) {
    5657        errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    57       } else if (n == 0) {
    58         prevOriginal = original;
    59         prevEstimated = estimated;
    60         n++;
    6158      } else {
    62         double err = Math.Abs(original - estimated);
    63         if ((original - prevOriginal) * (estimated - prevEstimated) >= 0.0) {
    64           correctSum += err;
     59        var actualEnumerator = actualContinuation.GetEnumerator();
     60        var predictedEnumerator = predictedContinuation.GetEnumerator();
     61        while (actualEnumerator.MoveNext() & predictedEnumerator.MoveNext() & errorState != OnlineCalculatorError.InvalidValueAdded) {
     62          double actual = actualEnumerator.Current;
     63          double predicted = predictedEnumerator.Current;
     64          if (double.IsNaN(actual) || double.IsNaN(predicted)) {
     65            errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
     66          } else {
     67            double err = Math.Abs(actual - predicted);
     68            // count as correct only if the trend (positive/negative/no change) is predicted correctly
     69            if ((actual - startValue) * (predicted - startValue) > 0.0 ||
     70              (actual - startValue).IsAlmost(predicted - startValue)) {
     71              correctSum += err;
     72            } else {
     73              incorrectSum += err;
     74            }
     75            n++;
     76          }
     77        }
     78        // check if both enumerators are at the end to make sure both enumerations have the same length
     79        if (actualEnumerator.MoveNext() || predictedEnumerator.MoveNext()) {
     80          errorState = errorState | OnlineCalculatorError.InvalidValueAdded;
    6581        } else {
    66           incorrectSum += err;
     82          errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded); // n >= 1
    6783        }
    68         n++;
    69         errorState = errorState & (~OnlineCalculatorError.InsufficientElementsAdded);        // n >= 1
    70         prevOriginal = original;
    71         prevEstimated = estimated;
    7284      }
    7385    }
    … …  
    8395
    8496
    85     public static double Calculate(IEnumerable<double> originalValues, IEnumerable<double> estimatedValues, out OnlineCalculatorError errorState) {
    86       IEnumerator<double> originalEnumerator = originalValues.GetEnumerator();
    87       IEnumerator<double> estimatedEnumerator = estimatedValues.GetEnumerator();
    88       OnlineWeightedDirectionalSymmetryCalculator dsCalculator = new OnlineWeightedDirectionalSymmetryCalculator();
     97    public static double Calculate(IEnumerable<double> startValues, IEnumerable<IEnumerable<double>> actualContinuations, IEnumerable<IEnumerable<double>> predictedContinuations, out OnlineCalculatorError errorState) {
     98      IEnumerator<double> startValueEnumerator = startValues.GetEnumerator();
     99      IEnumerator<IEnumerable<double>> actualContinuationsEnumerator = actualContinuations.GetEnumerator();
     100      IEnumerator<IEnumerable<double>> predictedContinuationsEnumerator = predictedContinuations.GetEnumerator();
     101      OnlineWeightedDirectionalSymmetryCalculator calculator = new OnlineWeightedDirectionalSymmetryCalculator();
    89102
    90       // add first element of time series as a reference point
    91       originalEnumerator.MoveNext();
    92       estimatedEnumerator.MoveNext();
    93       dsCalculator.Add(originalEnumerator.Current, estimatedEnumerator.Current);
    94 
    95       // always move forward both enumerators (do not use short-circuit evaluation!)
    96       while (originalEnumerator.MoveNext() & estimatedEnumerator.MoveNext()) {
    97         double original = originalEnumerator.Current;
    98         double estimated = estimatedEnumerator.Current;
    99         dsCalculator.Add(original, estimated);
    100         if (dsCalculator.ErrorState != OnlineCalculatorError.None) break;
     103      // always move forward all enumerators (do not use short-circuit evaluation!)
     104      while (startValueEnumerator.MoveNext() & actualContinuationsEnumerator.MoveNext() & predictedContinuationsEnumerator.MoveNext()) {
     105        calculator.Add(startValueEnumerator.Current, actualContinuationsEnumerator.Current, predictedContinuationsEnumerator.Current);
     106        if (calculator.ErrorState != OnlineCalculatorError.None) break;
    101107      }
    102108
    103       // check if both enumerators are at the end to make sure both enumerations have the same length
    104       if (dsCalculator.ErrorState == OnlineCalculatorError.None &&
    105           (originalEnumerator.MoveNext() || estimatedEnumerator.MoveNext())) {
    106         throw new ArgumentException("Number of elements in originalValues and estimatedValues enumerations doesn't match.");
     109      // check if all enumerators are at the end to make sure both enumerations have the same length
     110      if (calculator.ErrorState == OnlineCalculatorError.None &&
     111          (startValueEnumerator.MoveNext() || actualContinuationsEnumerator.MoveNext() || predictedContinuationsEnumerator.MoveNext())) {
     112        throw new ArgumentException("Number of elements in startValues, actualContinuations and estimatedValues predictedContinuations doesn't match.");
    107113      } else {
    108         errorState = dsCalculator.ErrorState;
    109         return dsCalculator.WeightedDirectionalSymmetry;
     114        errorState = calculator.ErrorState;
     115        return calculator.WeightedDirectionalSymmetry;
    110116      }
    111117    }
Note: See TracChangeset for help on using the changeset viewer.