Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
01/17/11 08:57:23 (14 years ago)
Author:
gkronber
Message:

worked on data analysis feature exploration branch. #1142

File:
1 edited

Legend:

Unmodified
Added
Removed

  • branches/DataAnalysis/HeuristicLab.Problems.DataAnalysis.MultiVariate.TimeSeriesPrognosis/3.3/Symbolic/Analyzer/ValidationBestScaledSymbolicTimeSeriesPrognosisSolutionAnalyzer.cs

    r5275 r5305  
    5353    private const string SymbolicExpressionTreeParameterName = "SymbolicExpressionTree";
    5454    private const string SymbolicExpressionTreeInterpreterParameterName = "SymbolicExpressionTreeInterpreter";
     55    private const string EvaluatorParameterName = "Evaluator";
     56    private const string MaximizationParameterName = "Maximization";
    5557    private const string ProblemDataParameterName = "ProblemData";
    5658    private const string ValidationSamplesStartParameterName = "SamplesStart";
    … …  
    6062    private const string UpperEstimationLimitParameterName = "UpperEstimationLimit";
    6163    private const string LowerEstimationLimitParameterName = "LowerEstimationLimit";
    62     private const string PredictionHorizonParameterName = "PredictionHorizon";
     64    private const string ValidationPredictionHorizonParameterName = "ValidationPredictionHorizon";
     65    private const string ModelPredictionHorizonParameterName = "ModelPredictionHorizon";
    6366    private const string ConditionVariableParameterName = "ConditionVariableName";
    64     private const string AlphaParameterName = "Alpha";
    65     private const string BetaParameterName = "Beta";
     67    private const string ResultsParameterName = "Results";
     68    private const string VariableFrequenciesParameterName = "VariableFrequencies";
     69    private const string RelativeNumberOfEvaluatedSamplesParameterName = "RelativeNumberOfEvaluatedSamples";
     70
    6671    private const string BestSolutionParameterName = "Best solution (validation)";
    6772    private const string BestSolutionQualityParameterName = "Best solution quality (validation)";
    6873    private const string CurrentBestValidationQualityParameterName = "Current best validation quality";
    6974    private const string BestSolutionQualityValuesParameterName = "Validation Quality";
    70     private const string ResultsParameterName = "Results";
    71     private const string VariableFrequenciesParameterName = "VariableFrequencies";
    7275    private const string BestKnownQualityParameterName = "BestKnownQuality";
    7376    private const string GenerationsParameterName = "Generations";
    7477
    75     private const string TrainingMeanSquaredErrorQualityParameterName = "Mean squared error (training)";
    76     private const string MinTrainingMeanSquaredErrorQualityParameterName = "Min mean squared error (training)";
    77     private const string MaxTrainingMeanSquaredErrorQualityParameterName = "Max mean squared error (training)";
    78     private const string AverageTrainingMeanSquaredErrorQualityParameterName = "Average mean squared error (training)";
    79     private const string BestTrainingMeanSquaredErrorQualityParameterName = "Best mean squared error (training)";
    80 
    81     private const string TrainingAverageRelativeErrorQualityParameterName = "Average relative error (training)";
    82     private const string MinTrainingAverageRelativeErrorQualityParameterName = "Min average relative error (training)";
    83     private const string MaxTrainingAverageRelativeErrorQualityParameterName = "Max average relative error (training)";
    84     private const string AverageTrainingAverageRelativeErrorQualityParameterName = "Average average relative error (training)";
    85     private const string BestTrainingAverageRelativeErrorQualityParameterName = "Best average relative error (training)";
    86 
    87     private const string TrainingRSquaredQualityParameterName = "R² (training)";
    88     private const string MinTrainingRSquaredQualityParameterName = "Min R² (training)";
    89     private const string MaxTrainingRSquaredQualityParameterName = "Max R² (training)";
    90     private const string AverageTrainingRSquaredQualityParameterName = "Average R² (training)";
    91     private const string BestTrainingRSquaredQualityParameterName = "Best R² (training)";
    92 
    93     private const string TestMeanSquaredErrorQualityParameterName = "Mean squared error (test)";
    94     private const string MinTestMeanSquaredErrorQualityParameterName = "Min mean squared error (test)";
    95     private const string MaxTestMeanSquaredErrorQualityParameterName = "Max mean squared error (test)";
    96     private const string AverageTestMeanSquaredErrorQualityParameterName = "Average mean squared error (test)";
    97     private const string BestTestMeanSquaredErrorQualityParameterName = "Best mean squared error (test)";
    98 
    99     private const string TestAverageRelativeErrorQualityParameterName = "Average relative error (test)";
    100     private const string MinTestAverageRelativeErrorQualityParameterName = "Min average relative error (test)";
    101     private const string MaxTestAverageRelativeErrorQualityParameterName = "Max average relative error (test)";
    102     private const string AverageTestAverageRelativeErrorQualityParameterName = "Average average relative error (test)";
    103     private const string BestTestAverageRelativeErrorQualityParameterName = "Best average relative error (test)";
    104 
    105     private const string TestRSquaredQualityParameterName = "R² (test)";
    106     private const string MinTestRSquaredQualityParameterName = "Min R² (test)";
    107     private const string MaxTestRSquaredQualityParameterName = "Max R² (test)";
    108     private const string AverageTestRSquaredQualityParameterName = "Average R² (test)";
    109     private const string BestTestRSquaredQualityParameterName = "Best R² (test)";
    110 
    111     private const string RSquaredValuesParameterName = "R²";
    112     private const string MeanSquaredErrorValuesParameterName = "Mean squared error";
    113     private const string RelativeErrorValuesParameterName = "Average relative error";
    114     private const string BestSolutionResultName = "Best solution (on validiation set)";
    115     private const string RelativeNumberOfEvaluatedSamplesParameterName = "RelativeNumberOfEvaluatedSamples";
     78    private const string BestSolutionMeanSquaredErrorTrainingParameterName = "Best validation solution mean squared error (training)";
     79    private const string BestSolutionMeanSquaredErrorTestParameterName = "Best validation solution mean squared error (test)";
     80    private const string BestSolutionRSquaredTrainingParameterName = "Best validation solution R² (training)";
     81    private const string BestSolutionRSquaredTestParameterName = "Best validation solution R² (test)";
     82    private const string BestSolutionDirectionalSymmetryTrainingParameterName = "Best validation solution directional symmetry (training)";
     83    private const string BestSolutionDirectionalSymmetryTestParameterName = "Best validation solution directional symmetry (test)";
     84    private const string BestSolutionTheilsUTrainingParameterName = "Best validation solution Theil's U (training)";
     85    private const string BestSolutionTheilsUTestParameterName = "Best validation solution Theil's U (test)";
     86    private const string BestSolutionTheilsUTrendTrainingParameterName = "Best validation solution Theil's U with trend (training)";
     87    private const string BestSolutionTheilsUTrendTestParameterName = "Best validation solution Theil's U with trend (test)";
    11688
    11789    #region parameter properties
    … …  
    132104    }
    133105    public ILookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator> EvaluatorParameter {
    134       get { return (ILookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator>)Parameters["Evaluator"]; }
     106      get { return (ILookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator>)Parameters[EvaluatorParameterName]; }
    135107    }
    136108    public IValueLookupParameter<IntValue> ValidationSamplesStartParameter {
    … …  
    146118      get { return (IValueLookupParameter<DoubleArray>)Parameters[LowerEstimationLimitParameterName]; }
    147119    }
    148     public IValueLookupParameter<IntValue> PredictionHorizonParameter {
    149       get { return (IValueLookupParameter<IntValue>)Parameters[PredictionHorizonParameterName]; }
     120    public IValueLookupParameter<IntValue> ValidationPredictionHorizonParameter {
     121      get { return (IValueLookupParameter<IntValue>)Parameters[ValidationPredictionHorizonParameterName]; }
     122    }
     123    public IValueLookupParameter<IntValue> ModelPredictionHorizonParameter {
     124      get { return (IValueLookupParameter<IntValue>)Parameters[ModelPredictionHorizonParameterName]; }
    150125    }
    151126    public ILookupParameter<SymbolicTimeSeriesPrognosisSolution> BestSolutionParameter {
    … …  
    171146    }
    172147    public ILookupParameter<BoolValue> MaximizationParameter {
    173       get { return (ILookupParameter<BoolValue>)Parameters["Maximization"]; }
     148      get { return (ILookupParameter<BoolValue>)Parameters[MaximizationParameterName]; }
     149    }
     150    public ILookupParameter<DoubleArray> BestSolutionMeanSquaredErrorTrainingParameter {
     151      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionMeanSquaredErrorTrainingParameterName]; }
     152    }
     153    public ILookupParameter<DoubleArray> BestSolutionMeanSquaredErrorTestParameter {
     154      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionMeanSquaredErrorTestParameterName]; }
     155    }
     156    public ILookupParameter<DoubleArray> BestSolutionRSquaredTrainingParameter {
     157      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionRSquaredTrainingParameterName]; }
     158    }
     159    public ILookupParameter<DoubleArray> BestSolutionRSquaredTestParameter {
     160      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionRSquaredTestParameterName]; }
     161    }
     162    public ILookupParameter<DoubleArray> BestSolutionDirectionalSymmetryTrainingParameter {
     163      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionDirectionalSymmetryTrainingParameterName]; }
     164    }
     165    public ILookupParameter<DoubleArray> BestSolutionDirectionalSymmetryTestParameter {
     166      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionDirectionalSymmetryTestParameterName]; }
     167    }
     168    public ILookupParameter<DoubleArray> BestSolutionTheilsUTrainingParameter {
     169      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTrainingParameterName]; }
     170    }
     171    public ILookupParameter<DoubleArray> BestSolutionTheilsUTestParameter {
     172      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTestParameterName]; }
     173    }
     174    public ILookupParameter<DoubleArray> BestSolutionTheilsUTrendTrainingParameter {
     175      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTrendTrainingParameterName]; }
     176    }
     177    public ILookupParameter<DoubleArray> BestSolutionTheilsUTrendTestParameter {
     178      get { return (ILookupParameter<DoubleArray>)Parameters[BestSolutionTheilsUTrendTestParameterName]; }
    174179    }
    175180    #endregion
    … …  
    199204      get { return LowerEstimationLimitParameter.ActualValue; }
    200205    }
    201     public IntValue PredictionHorizon {
    202       get { return PredictionHorizonParameter.ActualValue; }
     206    public IntValue ValidationPredictionHorizon {
     207      get { return ValidationPredictionHorizonParameter.ActualValue; }
     208    }
     209    public IntValue ModelPredictionHorizon {
     210      get { return ModelPredictionHorizonParameter.ActualValue; }
    203211    }
    204212    public StringValue ConditionVariableName {
    … …  
    219227    public BoolValue Maximization {
    220228      get { return MaximizationParameter.ActualValue; }
     229    }
     230    public DoubleArray BestSolutionMeanSquaredErrorTraining {
     231      get { return BestSolutionMeanSquaredErrorTrainingParameter.ActualValue; }
     232      set { BestSolutionMeanSquaredErrorTrainingParameter.ActualValue = value; }
     233    }
     234    public DoubleArray BestSolutionMeanSquaredErrorTest {
     235      get { return BestSolutionMeanSquaredErrorTestParameter.ActualValue; }
     236      set { BestSolutionMeanSquaredErrorTestParameter.ActualValue = value; }
     237    }
     238    public DoubleArray BestSolutionRSquaredTraining {
     239      get { return BestSolutionRSquaredTrainingParameter.ActualValue; }
     240      set { BestSolutionRSquaredTrainingParameter.ActualValue = value; }
     241    }
     242    public DoubleArray BestSolutionRSquaredTest {
     243      get { return BestSolutionRSquaredTestParameter.ActualValue; }
     244      set { BestSolutionRSquaredTestParameter.ActualValue = value; }
     245    }
     246    public DoubleArray BestSolutionDirectionalSymmetryTraining {
     247      get { return BestSolutionDirectionalSymmetryTrainingParameter.ActualValue; }
     248      set { BestSolutionDirectionalSymmetryTrainingParameter.ActualValue = value; }
     249    }
     250    public DoubleArray BestSolutionDirectionalSymmetryTest {
     251      get { return BestSolutionDirectionalSymmetryTestParameter.ActualValue; }
     252      set { BestSolutionDirectionalSymmetryTestParameter.ActualValue = value; }
     253    }
     254    public DoubleArray BestSolutionTheilsUTraining {
     255      get { return BestSolutionTheilsUTrainingParameter.ActualValue; }
     256      set { BestSolutionTheilsUTrainingParameter.ActualValue = value; }
     257    }
     258    public DoubleArray BestSolutionTheilsUTest {
     259      get { return BestSolutionTheilsUTestParameter.ActualValue; }
     260      set { BestSolutionTheilsUTestParameter.ActualValue = value; }
     261    }
     262    public DoubleArray BestSolutionTheilsUTrendTraining {
     263      get { return BestSolutionTheilsUTrendTrainingParameter.ActualValue; }
     264      set { BestSolutionTheilsUTrendTrainingParameter.ActualValue = value; }
     265    }
     266    public DoubleArray BestSolutionTheilsUTrendTest {
     267      get { return BestSolutionTheilsUTrendTestParameter.ActualValue; }
     268      set { BestSolutionTheilsUTrendTestParameter.ActualValue = value; }
    221269    }
    222270    #endregion
    … …  
    236284      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesStartParameterName, "The first index of the validation partition of the data set."));
    237285      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationSamplesEndParameterName, "The last index of the validation partition of the data set."));
    238       Parameters.Add(new ValueLookupParameter<IntValue>(PredictionHorizonParameterName, "The number of time steps for which to create a forecast."));
    239       Parameters.Add(new LookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator>("Evaluator", ""));
     286      Parameters.Add(new ValueLookupParameter<IntValue>(ValidationPredictionHorizonParameterName, "The number of time steps for which to create a forecast for the validation procedure."));
     287      Parameters.Add(new ValueLookupParameter<IntValue>(ModelPredictionHorizonParameterName, "Prediction horizont stored in the validation best model."));
     288      Parameters.Add(new LookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator>(EvaluatorParameterName, ""));
    240289      Parameters.Add(new ValueLookupParameter<DoubleArray>(UpperEstimationLimitParameterName, "The upper estimation limit that was set for the evaluation of the symbolic expression trees."));
    241290      Parameters.Add(new ValueLookupParameter<DoubleArray>(LowerEstimationLimitParameterName, "The lower estimation limit that was set for the evaluation of the symbolic expression trees."));
    … …  
    247296      Parameters.Add(new LookupParameter<DataTable>(VariableFrequenciesParameterName, "The variable frequencies table to use for the calculation of variable impacts"));
    248297      Parameters.Add(new ValueParameter<PercentValue>(RelativeNumberOfEvaluatedSamplesParameterName, "The relative number of samples of the dataset partition, which should be randomly chosen for evaluation between the start and end index.", new PercentValue(1)));
    249       Parameters.Add(new LookupParameter<BoolValue>("Maximization"));
     298      Parameters.Add(new LookupParameter<BoolValue>(MaximizationParameterName));
     299      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionMeanSquaredErrorTrainingParameterName));
     300      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionMeanSquaredErrorTestParameterName));
     301      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionRSquaredTrainingParameterName));
     302      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionRSquaredTestParameterName));
     303      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionDirectionalSymmetryTrainingParameterName));
     304      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionDirectionalSymmetryTestParameterName));
     305      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTrainingParameterName));
     306      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTestParameterName));
     307      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTrendTrainingParameterName));
     308      Parameters.Add(new LookupParameter<DoubleArray>(BestSolutionTheilsUTrendTestParameterName));
    250309    }
    251310
    … …  
    255314    [StorableHook(Persistence.Default.CompositeSerializers.Storable.HookType.AfterDeserialization)]
    256315    private void AfterDeserialization() {
    257       //if (!Parameters.ContainsKey("Evaluator")) {
    258       //  Parameters.Add(new LookupParameter<ISingleObjectiveSymbolicTimeSeriesPrognosisEvaluator>("Evaluator", ""));
    259       //}
     316
    260317    }
    261318
    … …  
    287344        double validationQuality;
    288345        validationQuality = evaluator.Evaluate(tree, ProblemData,
    289           SymbolicExpressionTreeInterpreter, rows, PredictionHorizon.Value, LowerEstimationLimit, UpperEstimationLimit);
     346          SymbolicExpressionTreeInterpreter, rows, ValidationPredictionHorizon.Value, LowerEstimationLimit, UpperEstimationLimit);
    290347        if ((Maximization.Value && validationQuality > bestValidationQuality) ||
    291348           (!Maximization.Value && validationQuality < bestValidationQuality)) {
    … …  
    304361        model.Description = "Best solution on validation partition found over the whole run.";
    305362
    306         var solution = new SymbolicTimeSeriesPrognosisSolution((MultiVariateDataAnalysisProblemData)ProblemData.Clone(), model, PredictionHorizon.Value, conditionalVariableName, LowerEstimationLimit.ToArray(), UpperEstimationLimit.ToArray());
     363        var solution = new SymbolicTimeSeriesPrognosisSolution((MultiVariateDataAnalysisProblemData)ProblemData.Clone(), model, ModelPredictionHorizon.Value, conditionalVariableName, LowerEstimationLimit.ToArray(), UpperEstimationLimit.ToArray());
    307364        solution.Name = BestSolutionParameterName;
    308365        solution.Description = "Best solution on validation partition found over the whole run.";
    … …  
    311368        BestSolutionQualityParameter.ActualValue = new DoubleValue(bestValidationQuality);
    312369
    313         // BestSymbolicTimeSeriesPrognosisSolutionAnalyzer.UpdateBestSolutionResults(solution, ProblemData, Results, Generations, VariableFrequencies);
     370        #region calculate accuracy
     371        List<string> targetVariables = ProblemData.TargetVariables.CheckedItems.Select(x => x.Value.Value).ToList();
     372
     373        // create a list of time series evaluators for each target variable
     374        Dictionary<string, List<IOnlineEvaluator>> trainingEvaluators =
     375          new Dictionary<string, List<IOnlineEvaluator>>();
     376        Dictionary<string, List<IOnlineEvaluator>> testEvaluators =
     377          new Dictionary<string, List<IOnlineEvaluator>>();
     378        foreach (string targetVariable in targetVariables) {
     379          trainingEvaluators.Add(targetVariable, new List<IOnlineEvaluator>());
     380          trainingEvaluators[targetVariable].Add(new OnlineMeanSquaredErrorEvaluator());
     381          trainingEvaluators[targetVariable].Add(new OnlinePearsonsRSquaredEvaluator());
     382          trainingEvaluators[targetVariable].Add(new OnlineDirectionalSymmetryEvaluator());
     383          trainingEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator());
     384          trainingEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator(10));
     385
     386          testEvaluators.Add(targetVariable, new List<IOnlineEvaluator>());
     387          testEvaluators[targetVariable].Add(new OnlineMeanSquaredErrorEvaluator());
     388          testEvaluators[targetVariable].Add(new OnlinePearsonsRSquaredEvaluator());
     389          testEvaluators[targetVariable].Add(new OnlineDirectionalSymmetryEvaluator());
     390          testEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator());
     391          testEvaluators[targetVariable].Add(new OnlineTheilsUStatisticEvaluator(10));
     392        }
     393
     394        Evaluate(solution, solution.ProblemData.Dataset, targetVariables, conditionalVariableName, trainingStart, trainingEnd, trainingEvaluators);
     395        Evaluate(solution, solution.ProblemData.Dataset, targetVariables, conditionalVariableName, testStart, testEnd, testEvaluators);
     396        #endregion
     397        BestSolutionMeanSquaredErrorTraining = new DoubleArray((from variable in targetVariables
     398                                                                let eval = trainingEvaluators[variable].OfType<OnlineMeanSquaredErrorEvaluator>().Single()
     399                                                                select TryGetValue(eval))
     400                                                                .ToArray());
     401        BestSolutionMeanSquaredErrorTest = new DoubleArray((from variable in targetVariables
     402                                                            select TryGetValue(testEvaluators[variable].OfType<OnlineMeanSquaredErrorEvaluator>().Single()))
     403                                                          .ToArray());
     404        BestSolutionRSquaredTraining = new DoubleArray((from variable in targetVariables
     405                                                        select TryGetValue(trainingEvaluators[variable].OfType<OnlinePearsonsRSquaredEvaluator>().Single()))
     406                                                                .ToArray());
     407        BestSolutionRSquaredTest = new DoubleArray((from variable in targetVariables
     408                                                    select TryGetValue(testEvaluators[variable].OfType<OnlinePearsonsRSquaredEvaluator>().Single()))
     409                                                          .ToArray());
     410        BestSolutionDirectionalSymmetryTraining = new DoubleArray((from variable in targetVariables
     411                                                                   select TryGetValue(trainingEvaluators[variable].OfType<OnlineDirectionalSymmetryEvaluator>().Single()))
     412                                                                .ToArray());
     413        BestSolutionDirectionalSymmetryTest = new DoubleArray((from variable in targetVariables
     414                                                               select TryGetValue(testEvaluators[variable].OfType<OnlineDirectionalSymmetryEvaluator>().Single()))
     415                                                          .ToArray());
     416        BestSolutionTheilsUTraining = new DoubleArray((from variable in targetVariables
     417                                                       select TryGetValue(trainingEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().First()))
     418                                                                .ToArray());
     419        BestSolutionTheilsUTest = new DoubleArray((from variable in targetVariables
     420                                                   select TryGetValue(testEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().First()))
     421                                                          .ToArray());
     422        BestSolutionTheilsUTrendTraining = new DoubleArray((from variable in targetVariables
     423                                                            select TryGetValue(trainingEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().Skip(1).First()))
     424                                                                .ToArray());
     425        BestSolutionTheilsUTrendTest = new DoubleArray((from variable in targetVariables
     426                                                        select TryGetValue(testEvaluators[variable].OfType<OnlineTheilsUStatisticEvaluator>().Skip(1).First()))
     427                                                          .ToArray());
     428        if (!Results.ContainsKey(BestSolutionParameterName)) {
     429          for (int i = 0; i < targetVariables.Count; i++) {
     430            Results.Add(new Result(BestSolutionMeanSquaredErrorTrainingParameterName + " (" + targetVariables[i] + ")",
     431              new DoubleValue(BestSolutionMeanSquaredErrorTraining[i])));
     432            Results.Add(new Result(BestSolutionMeanSquaredErrorTestParameterName + " (" + targetVariables[i] + ")",
     433              new DoubleValue(BestSolutionMeanSquaredErrorTest[i])));
     434            Results.Add(new Result(BestSolutionRSquaredTrainingParameterName + " (" + targetVariables[i] + ")",
     435              new DoubleValue(BestSolutionRSquaredTraining[i])));
     436            Results.Add(new Result(BestSolutionRSquaredTestParameterName + " (" + targetVariables[i] + ")",
     437              new DoubleValue(BestSolutionRSquaredTest[i])));
     438            Results.Add(new Result(BestSolutionDirectionalSymmetryTrainingParameterName + " (" + targetVariables[i] + ")",
     439              new DoubleValue(BestSolutionDirectionalSymmetryTraining[i])));
     440            Results.Add(new Result(BestSolutionDirectionalSymmetryTestParameterName + " (" + targetVariables[i] + ")",
     441              new DoubleValue(BestSolutionDirectionalSymmetryTest[i])));
     442            Results.Add(new Result(BestSolutionTheilsUTrainingParameterName + " (" + targetVariables[i] + ")",
     443              new DoubleValue(BestSolutionTheilsUTraining[i])));
     444            Results.Add(new Result(BestSolutionTheilsUTestParameterName + " (" + targetVariables[i] + ")",
     445              new DoubleValue(BestSolutionTheilsUTest[i])));
     446            Results.Add(new Result(BestSolutionTheilsUTrendTrainingParameterName + " (" + targetVariables[i] + ")",
     447              new DoubleValue(BestSolutionTheilsUTrendTraining[i])));
     448            Results.Add(new Result(BestSolutionTheilsUTrendTestParameterName + " (" + targetVariables[i] + ")",
     449              new DoubleValue(BestSolutionTheilsUTrendTest[i])));
     450          }
     451        } else {
     452          for (int i = 0; i < targetVariables.Count; i++) {
     453            Results[BestSolutionMeanSquaredErrorTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
     454              new DoubleValue(BestSolutionMeanSquaredErrorTraining[i]);
     455            Results[BestSolutionMeanSquaredErrorTestParameterName + " (" + targetVariables[i] + ")"].Value =
     456              new DoubleValue(BestSolutionMeanSquaredErrorTest[i]);
     457            Results[BestSolutionRSquaredTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
     458              new DoubleValue(BestSolutionRSquaredTraining[i]);
     459            Results[BestSolutionRSquaredTestParameterName + " (" + targetVariables[i] + ")"].Value =
     460              new DoubleValue(BestSolutionRSquaredTest[i]);
     461            Results[BestSolutionDirectionalSymmetryTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
     462              new DoubleValue(BestSolutionDirectionalSymmetryTraining[i]);
     463            Results[BestSolutionDirectionalSymmetryTestParameterName + " (" + targetVariables[i] + ")"].Value =
     464              new DoubleValue(BestSolutionDirectionalSymmetryTest[i]);
     465            Results[BestSolutionTheilsUTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
     466              new DoubleValue(BestSolutionTheilsUTraining[i]);
     467            Results[BestSolutionTheilsUTestParameterName + " (" + targetVariables[i] + ")"].Value =
     468              new DoubleValue(BestSolutionTheilsUTest[i]);
     469            Results[BestSolutionTheilsUTrendTrainingParameterName + " (" + targetVariables[i] + ")"].Value =
     470              new DoubleValue(BestSolutionTheilsUTrendTraining[i]);
     471            Results[BestSolutionTheilsUTrendTestParameterName + " (" + targetVariables[i] + ")"].Value =
     472              new DoubleValue(BestSolutionTheilsUTrendTest[i]);
     473          }
     474        }
    314475      }
    315476
    316477      if (!Results.ContainsKey(BestSolutionQualityValuesParameterName)) {
    317         Results.Add(new Result(BestSolutionResultName, BestSolutionParameter.ActualValue));
     478        Results.Add(new Result(BestSolutionParameterName, BestSolutionParameter.ActualValue));
    318479        Results.Add(new Result(BestSolutionQualityValuesParameterName, new DataTable(BestSolutionQualityValuesParameterName, BestSolutionQualityValuesParameterName)));
    319480        Results.Add(new Result(BestSolutionQualityParameterName, new DoubleValue()));
    320481        Results.Add(new Result(CurrentBestValidationQualityParameterName, new DoubleValue()));
    321482      }
    322       Results[BestSolutionResultName].Value = BestSolutionParameter.ActualValue;
     483      Results[BestSolutionParameterName].Value = BestSolutionParameter.ActualValue;
    323484      Results[BestSolutionQualityParameterName].Value = new DoubleValue(BestSolutionQualityParameter.ActualValue.Value);
    324485      Results[CurrentBestValidationQualityParameterName].Value = new DoubleValue(bestValidationQuality);
    … …  
    329490      #endregion
    330491      return base.Apply();
     492    }
     493
     494    private double TryGetValue(IOnlineEvaluator eval) {
     495      try {
     496        return eval.Value;
     497      }
     498      catch {
     499        return double.NaN;
     500      }
    331501    }
    332502
    … …  
    364534    }
    365535
    366 
     536    private void Evaluate(SymbolicTimeSeriesPrognosisSolution solution, Dataset dataset, IEnumerable<string> targetVariables, string conditionalEvaluationVariable, int start, int end, Dictionary<string, List<IOnlineEvaluator>> evaluators) {
     537
     538      for (int row = start; row < end; row++) {
     539        if (string.IsNullOrEmpty(conditionalEvaluationVariable) || dataset[conditionalEvaluationVariable, row] != 0) {
     540          // prepare evaluators for each target variable for a new prediction window
     541          foreach (var entry in evaluators) {
     542            double referenceOriginalValue = dataset[entry.Key, row - 1];
     543            foreach (IOnlineTimeSeriesPrognosisEvaluator evaluator in entry.Value.OfType<IOnlineTimeSeriesPrognosisEvaluator>()) {
     544              evaluator.StartNewPredictionWindow(referenceOriginalValue);
     545            }
     546          }
     547
     548
     549          if (string.IsNullOrEmpty(conditionalEvaluationVariable) ||
     550            dataset[conditionalEvaluationVariable, row] > 0) {
     551            int timestep = 0;
     552            foreach (double[] x in solution.GetPrognosis(row)) {
     553              int targetIndex = 0;
     554              if (row + timestep < dataset.Rows) {
     555                foreach (var targetVariable in targetVariables) {
     556                  double originalValue = dataset[targetVariable, row + timestep];
     557                  double estimatedValue = x[targetIndex];
     558                  if (IsValidValue(originalValue) && IsValidValue(estimatedValue)) {
     559                    foreach (IOnlineEvaluator evaluator in evaluators[targetVariable]) {
     560                      evaluator.Add(originalValue, estimatedValue);
     561                    }
     562                  }
     563                  targetIndex++;
     564                }
     565              }
     566              timestep++;
     567            }
     568          }
     569        }
     570      }
     571    }
     572    private bool IsValidValue(double d) {
     573      return !(double.IsNaN(d) || double.IsInfinity(d));
     574    }
    367575
    368576    private static void AddValue(DataTable table, double data, string name, string description) {
Note: See TracChangeset for help on using the changeset viewer.