Changeset 7183 for branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis

Timestamp:

12/13/11 13:52:22 (13 years ago)

Author:

gkronber

Message:

#1081 implemented remaining metrics for time series prognosis solutions, added estimation limits fixed training and validation best solution analyzers, implemented overfitting analyzer.

File:

• branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisSolutionBase.cs (modified) (6 diffs)

branches/HeuristicLab.TimeSeries/HeuristicLab.Problems.DataAnalysis/3.4/Implementation/TimeSeriesPrognosis/TimeSeriesPrognosisSolutionBase.cs

@@ -20 +20 @@
 #endregion
 
+using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
@@ -195 +196 @@
     private void AfterDeserialization() {
       if (horizon == 0) horizon = 1;
+      bool anyNewResult = false;
+      if (!ContainsKey(TrainingTheilsUStatisticLastResultName)) {
+        Add(new Result(TrainingTheilsUStatisticLastResultName, "The average Theil's U statistic (reference: previous value) of the forecasts of the model on the training partition", new DoubleArray()));
+        anyNewResult = true;
+      }
+      if (!ContainsKey(TestTheilsUStatisticLastResultName)) {
+        Add(new Result(TestTheilsUStatisticLastResultName, "The average Theil's U statistic (reference: previous value) of the forecasts of the model on the test partition", new DoubleArray()));
+        anyNewResult = true;
+      }
+      if (!ContainsKey(TrainingTheilsUStatisticMeanResultName)) {
+        Add(new Result(TrainingTheilsUStatisticMeanResultName, "The average Theil's U statistic (reference: mean value) of the forecasts of the model on the training partition", new DoubleArray()));
+        anyNewResult = true;
+      }
+      if (!ContainsKey(TestTheilsUStatisticMeanResultName)) {
+        Add(new Result(TestTheilsUStatisticMeanResultName, "The average Theil's U statistic (reference: mean value) of the forecasts of the model on the test partition", new DoubleArray()));
+        anyNewResult = true;
+      }
+      if (!ContainsKey(TrainingTheilsUStatisticMAResultName)) {
+        Add(new Result(TrainingTheilsUStatisticMAResultName, "The average Theil's U statistic (reference: moving average) of the forecasts of the model on the training partition", new DoubleArray()));
+        anyNewResult = true;
+      }
+      if (!ContainsKey(TestTheilsUStatisticMAResultName)) {
+        Add(new Result(TestTheilsUStatisticMAResultName, "The average Theil's U statistic (reference: moving average) of the forecasts of the model on the test partition", new DoubleArray()));
+        anyNewResult = true;
+      }
+      if (anyNewResult)
+        RecalculateResults();
     }
 
@@ -240 +268 @@
       //double[] trainingTheilsU = new double[targetVariables.Length];
       //double[] testTheilsU = new double[targetVariables.Length];
+
+      var trainingMseCalculators = new OnlineMeanSquaredErrorCalculator[targetVariables.Length];
+      var testMseCalculators = new OnlineMeanSquaredErrorCalculator[targetVariables.Length];
+      var trainingMaeCalculators = new OnlineMeanAbsoluteErrorCalculator[targetVariables.Length];
+      var testMaeCalculators = new OnlineMeanAbsoluteErrorCalculator[targetVariables.Length];
+      var trainingRSquaredCalculators = new OnlinePearsonsRSquaredCalculator[targetVariables.Length];
+      var testRSquaredCalculators = new OnlinePearsonsRSquaredCalculator[targetVariables.Length];
+      var trainingRelErrorCalculators = new OnlineMeanAbsolutePercentageErrorCalculator[targetVariables.Length];
+      var testRelErrorCalculators = new OnlineMeanAbsolutePercentageErrorCalculator[targetVariables.Length];
+      var trainingNmseCalculators = new OnlineNormalizedMeanSquaredErrorCalculator[targetVariables.Length];
+      var testNmseCalculators = new OnlineNormalizedMeanSquaredErrorCalculator[targetVariables.Length];
 
       var trainingDsCalculators = new OnlineDirectionalSymmetryCalculator[targetVariables.Length];
@@ -252 +291 @@
       var testTheilsUMovingAverageCalculators = new OnlineTheilsUStatisticCalculator[targetVariables.Length];
       for (int i = 0; i < targetVariables.Length; i++) {
+        trainingMseCalculators[i] = new OnlineMeanSquaredErrorCalculator();
+        testMseCalculators[i] = new OnlineMeanSquaredErrorCalculator();
+        trainingMaeCalculators[i] = new OnlineMeanAbsoluteErrorCalculator();
+        testMaeCalculators[i] = new OnlineMeanAbsoluteErrorCalculator();
+        trainingRSquaredCalculators[i] = new OnlinePearsonsRSquaredCalculator();
+        testRSquaredCalculators[i] = new OnlinePearsonsRSquaredCalculator();
+        trainingRelErrorCalculators[i] = new OnlineMeanAbsolutePercentageErrorCalculator();
+        testRelErrorCalculators[i] = new OnlineMeanAbsolutePercentageErrorCalculator();
+        trainingNmseCalculators[i] = new OnlineNormalizedMeanSquaredErrorCalculator();
+        testNmseCalculators[i] = new OnlineNormalizedMeanSquaredErrorCalculator();
+
         trainingDsCalculators[i] = new OnlineDirectionalSymmetryCalculator();
         testDsCalculators[i] = new OnlineDirectionalSymmetryCalculator();
@@ -286 +336 @@
             trainingTheilsUMeanCalculators[t].Add(startValue, actualContinuation.Select(x => mean), actualContinuation, prognosedContinuation);
             trainingTheilsUMovingAverageCalculators[t].Add(startValue, movingAverageContinuation, actualContinuation, prognosedContinuation);
+
+            var actualContinuationEnumerator = actualContinuation.GetEnumerator();
+            var prognosedContinuationEnumerator = prognosedContinuation.GetEnumerator();
+            while (actualContinuationEnumerator.MoveNext() & prognosedContinuationEnumerator.MoveNext()) {
+              trainingMseCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              trainingMaeCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              trainingRelErrorCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              trainingRSquaredCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              trainingNmseCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+            }
+            if (actualContinuationEnumerator.MoveNext() | prognosedContinuationEnumerator.MoveNext())
+              throw new ArgumentException(
+                "Different number of elements in Actual continuation and prognosed continuation.");
           }
         }
@@ -312 +375 @@
             testTheilsUMeanCalculators[t].Add(startValue, actualContinuation.Select(x => mean), actualContinuation, prognosedContinuation);
             testTheilsUMovingAverageCalculators[t].Add(startValue, movingAverageContinuation, actualContinuation, prognosedContinuation);
+
+            var actualContinuationEnumerator = actualContinuation.GetEnumerator();
+            var prognosedContinuationEnumerator = prognosedContinuation.GetEnumerator();
+            while (actualContinuationEnumerator.MoveNext() & prognosedContinuationEnumerator.MoveNext()) {
+              testMseCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              testMaeCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              testRelErrorCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              testRSquaredCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+              testNmseCalculators[t].Add(actualContinuationEnumerator.Current, prognosedContinuationEnumerator.Current);
+            }
+            if (actualContinuationEnumerator.MoveNext() | prognosedContinuationEnumerator.MoveNext())
+              throw new ArgumentException(
+                "Different number of elements in Actual continuation and prognosed continuation.");
           }
         }
       }
 
+
+      TrainingMeanSquaredError = trainingMseCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+      TestMeanSquaredError = testMseCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+      TrainingMeanAbsoluteError = trainingMaeCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+      TestMeanAbsoluteError = testMaeCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+      TrainingRelativeError = trainingRelErrorCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+      TestRelativeError = testRelErrorCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+      TrainingRSquared = trainingRSquaredCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : 0.0)
+        .ToArray();
+      TestRSquared = testRSquaredCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : 0.0)
+        .ToArray();
+      TrainingNormalizedMeanSquaredError = trainingNmseCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+      TestNormalizedMeanSquaredError = testNmseCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : double.PositiveInfinity)
+        .ToArray();
+
       TrainingDirectionalSymmetry = trainingDsCalculators.Select(c => c.ErrorState == OnlineCalculatorError.None ? c.Value : 0.0)
         .ToArray();