Ignore:
Timestamp:
11/09/11 10:59:31 (8 years ago)
Author:
gkronber
Message:

#1081: cleared up definition of accuracy metrics for time series prognosis to make the distinction between one n-step forecast and n one-step forecasts clearer. Implemented calculators for time series accuracy metrics to support the calculation of the average accuracy over m n-step forecasts and adapted the unit tests accordingly.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/sources/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisSolutionBase.cs

    r6961 r6974  
    4040    private const string TrainingNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (training)";
    4141    private const string TestNormalizedMeanSquaredErrorResultName = "Normalized mean squared error (test)";
    42     private const string TrainingDirectionalSymmetryResultName = "Directional symmetry (training)";
    43     private const string TestDirectionalSymmetryResultName = "Directional symmetry (test)";
    44     private const string TrainingWeightedDirectionalSymmetryResultName = "Weighted directional symmetry (training)";
    45     private const string TestWeightedDirectionalSymmetryResultName = "Weighted directional symmetry (test)";
    46     private const string TrainingTheilsUStatisticResultName = "Theil's U (training)";
    47     private const string TestTheilsUStatisticResultName = "Theil's U (test)";
     42    private const string TrainingDirectionalSymmetryResultName = "Average directional symmetry (training)";
     43    private const string TestDirectionalSymmetryResultName = "Average directional symmetry (test)";
     44    private const string TrainingWeightedDirectionalSymmetryResultName = "Average weighted directional symmetry (training)";
     45    private const string TestWeightedDirectionalSymmetryResultName = "Average weighted directional symmetry (test)";
     46    private const string TrainingTheilsUStatisticResultName = "Average Theil's U (training)";
     47    private const string TestTheilsUStatisticResultName = "Average Theil's U (test)";
    4848
    4949    public new ITimeSeriesPrognosisModel Model {
     
    146146      Add(new Result(TrainingNormalizedMeanSquaredErrorResultName, "Normalized mean of squared errors of the model on the training partition", new DoubleValue()));
    147147      Add(new Result(TestNormalizedMeanSquaredErrorResultName, "Normalized mean of squared errors of the model on the test partition", new DoubleValue()));
    148       Add(new Result(TrainingDirectionalSymmetryResultName, "The directional symmetry of the output of the model on the training partition", new DoubleValue()));
    149       Add(new Result(TestDirectionalSymmetryResultName, "The directional symmetry of the output of the model on the test partition", new DoubleValue()));
    150       Add(new Result(TrainingWeightedDirectionalSymmetryResultName, "The weighted directional symmetry of the output of the model on the training partition", new DoubleValue()));
    151       Add(new Result(TestWeightedDirectionalSymmetryResultName, "The weighted directional symmetry of the output of the model on the test partition", new DoubleValue()));
    152       Add(new Result(TrainingTheilsUStatisticResultName, "The Theil's U statistic of the output of the model on the training partition", new DoubleValue()));
    153       Add(new Result(TestTheilsUStatisticResultName, "The Theil's U statistic of the output of the model on the test partition", new DoubleValue()));
     148      Add(new Result(TrainingDirectionalSymmetryResultName, "The average directional symmetry of the forecasts of the model on the training partition", new PercentValue()));
     149      Add(new Result(TestDirectionalSymmetryResultName, "The average directional symmetry of the forecasts of the model on the test partition", new PercentValue()));
     150      Add(new Result(TrainingWeightedDirectionalSymmetryResultName, "The average weighted directional symmetry of the forecasts of the model on the training partition", new DoubleValue()));
     151      Add(new Result(TestWeightedDirectionalSymmetryResultName, "The average weighted directional symmetry of the forecasts of the model on the test partition", new DoubleValue()));
     152      Add(new Result(TrainingTheilsUStatisticResultName, "The average Theil's U statistic of the forecasts of the model on the training partition", new DoubleValue()));
     153      Add(new Result(TestTheilsUStatisticResultName, "The average Theil's U statistic of the forecasts of the model on the test partition", new DoubleValue()));
    154154    }
    155155
     
    191191      TestNormalizedMeanSquaredError = errorState == OnlineCalculatorError.None ? testNmse : double.NaN;
    192192
    193       double trainingDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
     193      var startTrainingValues = originalTrainingValues;
     194      // each continuation is only one element long
     195      var actualContinuationsTraining = from x in originalTrainingValues.Skip(1)
     196                                        select Enumerable.Repeat(x, 1);
     197      // each forecast is only one elemnt long
     198      // disregards the first estimated value (we could include this again by extending the list of original values by one step to the left
     199      // this is the easier way
     200      var predictedContinuationsTraining = from x in estimatedTrainingValues.Skip(1)
     201                                           select Enumerable.Repeat(x, 1);
     202
     203      var startTestValues = originalTestValues;
     204      var actualContinuationsTest = from x in originalTestValues.Skip(1)
     205                                    select Enumerable.Repeat(x, 1);
     206      var predictedContinuationsTest = from x in estimatedTestValues.Skip(1)
     207                                       select Enumerable.Repeat(x, 1);
     208
     209      double trainingDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(startTrainingValues, actualContinuationsTraining, predictedContinuationsTraining, out errorState);
    194210      TrainingDirectionalSymmetry = errorState == OnlineCalculatorError.None ? trainingDirectionalSymmetry : double.NaN;
    195       double testDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
     211      double testDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(startTestValues, actualContinuationsTest, predictedContinuationsTest, out errorState);
    196212      TestDirectionalSymmetry = errorState == OnlineCalculatorError.None ? testDirectionalSymmetry : double.NaN;
    197213
    198       double trainingWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
     214      double trainingWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(startTrainingValues, actualContinuationsTraining, predictedContinuationsTraining, out errorState);
    199215      TrainingWeightedDirectionalSymmetry = errorState == OnlineCalculatorError.None ? trainingWeightedDirectionalSymmetry : double.NaN;
    200       double testWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
     216      double testWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(startTestValues, actualContinuationsTest, predictedContinuationsTest, out errorState);
    201217      TestWeightedDirectionalSymmetry = errorState == OnlineCalculatorError.None ? testWeightedDirectionalSymmetry : double.NaN;
    202218
    203       double trainingTheilsU = OnlineTheilsUStatisticCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
     219      double trainingTheilsU = OnlineTheilsUStatisticCalculator.Calculate(startTrainingValues, actualContinuationsTraining, predictedContinuationsTraining, out errorState);
    204220      TrainingTheilsUStatistic = errorState == OnlineCalculatorError.None ? trainingTheilsU : double.NaN;
    205       double testTheilsU = OnlineTheilsUStatisticCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
     221      double testTheilsU = OnlineTheilsUStatisticCalculator.Calculate(startTestValues, actualContinuationsTest, predictedContinuationsTest, out errorState);
    206222      TestTheilsUStatistic = errorState == OnlineCalculatorError.None ? testTheilsU : double.NaN;
    207 
    208 
    209223    }
    210224  }
Note: See TracChangeset for help on using the changeset viewer.