Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Modeling/3.2/ModelingResultCalculators.cs @ 2397

Last change on this file since 2397 was 2397, checked in by mkommend, 15 years ago

corrected ModelingResultCalculators for TimeSeriesPrognosis (ticket #755)
File size: 14.0 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2008 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Text;
26using HeuristicLab.Core;
27
28namespace HeuristicLab.Modeling {
29  public abstract class ModelingResultCalculators {
30    private enum DatasetPart { Training, Validation, Test };
31
32    private static readonly Dictionary<ModelingResult, Func<double[,], double>> ClassificationModelingResults;
33    private static readonly Dictionary<ModelingResult, Func<double[,], double>> RegressionModelingResults;
34    private static readonly Dictionary<ModelingResult, Func<double[,], double>> TimeSeriesPrognosisModelingResults;
35    private static readonly Dictionary<ModelingResult, IOperator> ClassificationModelingResultEvaluators;
36    private static readonly Dictionary<ModelingResult, IOperator> RegressionModelingResultEvaluators;
37    private static readonly Dictionary<ModelingResult, IOperator> TimeSeriesPrognosisModelingResultEvaluators;
38
39    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> regressionResults =
40      new Dictionary<Type, IEnumerable<ModelingResult>>() {
41        { typeof(SimpleMSEEvaluator),
42          new ModelingResult[] {
43            ModelingResult.TrainingMeanSquaredError,
44            ModelingResult.ValidationMeanSquaredError,
45            ModelingResult.TestMeanSquaredError
46          }},
47        { typeof(SimpleNMSEEvaluator),
48          new ModelingResult[] {
49            ModelingResult.TrainingNormalizedMeanSquaredError,
50            ModelingResult.ValidationNormalizedMeanSquaredError,
51            ModelingResult.TestNormalizedMeanSquaredError
52          }
53        },
54        { typeof(SimpleR2Evaluator),
55          new ModelingResult[] {
56            ModelingResult.TrainingCoefficientOfDetermination,
57            ModelingResult.ValidationCoefficientOfDetermination,
58            ModelingResult.TestCoefficientOfDetermination
59          }
60        },
61        { typeof(SimpleVarianceAccountedForEvaluator),
62          new ModelingResult[] {
63            ModelingResult.TrainingVarianceAccountedFor,
64            ModelingResult.ValidationVarianceAccountedFor,
65            ModelingResult.TestVarianceAccountedFor
66          }
67        },
68        { typeof(SimpleMeanAbsolutePercentageErrorEvaluator),
69          new ModelingResult[] {
70            ModelingResult.TrainingMeanAbsolutePercentageError,
71            ModelingResult.ValidationMeanAbsolutePercentageError,
72            ModelingResult.TestMeanAbsolutePercentageError
73          }
74        },
75        { typeof(SimpleMeanAbsolutePercentageOfRangeErrorEvaluator),
76          new ModelingResult[] {
77            ModelingResult.TrainingMeanAbsolutePercentageOfRangeError,
78            ModelingResult.ValidationMeanAbsolutePercentageOfRangeError,
79            ModelingResult.TestMeanAbsolutePercentageOfRangeError
80          }
81        }
82      };
83
84    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> timeSeriesResults =
85      new Dictionary<Type, IEnumerable<ModelingResult>>() {
86        { typeof(SimpleTheilInequalityCoefficientEvaluator),
87          new ModelingResult[] {
88            ModelingResult.TrainingTheilInequality,
89            ModelingResult.ValidationTheilInequality,
90            ModelingResult.TestTheilInequality
91          }
92        },
93        { typeof(SimpleDirectionalSymmetryEvaluator),
94          new ModelingResult[] {
95            ModelingResult.TrainingDirectionalSymmetry,
96            ModelingResult.ValidationDirectionalSymmetry,
97            ModelingResult.TestDirectionalSymmetry
98          }
99        },
100        { typeof(SimpleWeightedDirectionalSymmetryEvaluator),
101          new ModelingResult[] {
102            ModelingResult.TrainingWeightedDirectionalSymmetry,
103            ModelingResult.ValidationWeightedDirectionalSymmetry,
104            ModelingResult.TestWeightedDirectionalSymmetry
105          }
106        }
107      };
108
109    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> classificationResults =
110      new Dictionary<Type, IEnumerable<ModelingResult>>() {
111        { typeof(SimpleAccuracyEvaluator),
112          new ModelingResult[] {
113            ModelingResult.TrainingAccuracy,
114            ModelingResult.ValidationAccuracy,
115            ModelingResult.TestAccuracy
116          }
117        }
118      };
119
120
121    static ModelingResultCalculators() {
122      RegressionModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
123      ClassificationModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
124      TimeSeriesPrognosisModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
125
126      //Mean squared errors
127      RegressionModelingResults[ModelingResult.TrainingMeanSquaredError] = SimpleMSEEvaluator.Calculate;
128      RegressionModelingResults[ModelingResult.ValidationMeanSquaredError] = SimpleMSEEvaluator.Calculate;
129      RegressionModelingResults[ModelingResult.TestMeanSquaredError] = SimpleMSEEvaluator.Calculate;
130
131      //Normalized mean squared errors
132      RegressionModelingResults[ModelingResult.TrainingNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
133      RegressionModelingResults[ModelingResult.ValidationNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
134      RegressionModelingResults[ModelingResult.TestNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
135
136      //Mean absolute percentage error
137      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
138      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
139      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
140
141      //Mean absolute percentage of range error
142      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
143      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
144      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
145
146      //Coefficient of determination
147      RegressionModelingResults[ModelingResult.TrainingCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
148      RegressionModelingResults[ModelingResult.ValidationCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
149      RegressionModelingResults[ModelingResult.TestCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
150
151      //Variance accounted for
152      RegressionModelingResults[ModelingResult.TrainingVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
153      RegressionModelingResults[ModelingResult.ValidationVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
154      RegressionModelingResults[ModelingResult.TestVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
155
156      //Accuracy
157      ClassificationModelingResults[ModelingResult.TrainingAccuracy] = SimpleAccuracyEvaluator.Calculate;
158      ClassificationModelingResults[ModelingResult.ValidationAccuracy] = SimpleAccuracyEvaluator.Calculate;
159      ClassificationModelingResults[ModelingResult.TestAccuracy] = SimpleAccuracyEvaluator.Calculate;
160
161      //Theil inequality
162      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
163      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
164      TimeSeriesPrognosisModelingResults[ModelingResult.TestTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
165
166      //Directional symmetry
167      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
168      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
169      TimeSeriesPrognosisModelingResults[ModelingResult.TestDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
170
171      //Weighted directional symmetry
172      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
173      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
174      TimeSeriesPrognosisModelingResults[ModelingResult.TestWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
175
176      #region result evaluators
177
178      RegressionModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
179      foreach (Type evaluatorT in regressionResults.Keys) {
180        foreach (ModelingResult r in regressionResults[evaluatorT]) {
181          RegressionModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
182        }
183      }
184
185      timeSeriesResults = CombineDictionaries(regressionResults, timeSeriesResults);
186      TimeSeriesPrognosisModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
187      foreach (Type evaluatorT in timeSeriesResults.Keys) {
188        foreach (ModelingResult r in timeSeriesResults[evaluatorT]) {
189          TimeSeriesPrognosisModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
190        }
191      }
192
193      classificationResults = CombineDictionaries(regressionResults, classificationResults);
194      ClassificationModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
195      foreach (Type evaluatorT in classificationResults.Keys) {
196        foreach (ModelingResult r in classificationResults[evaluatorT]) {
197          ClassificationModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
198        }
199      }
200
201      #endregion
202    }
203
204    public static Dictionary<ModelingResult, Func<double[,], double>> GetModelingResult(ModelType modelType) {
205      switch (modelType) {
206        case ModelType.Regression:
207          return CombineDictionaries(RegressionModelingResults, new Dictionary<ModelingResult, Func<double[,], double>>());
208        case ModelType.Classification:
209          return CombineDictionaries(RegressionModelingResults, ClassificationModelingResults);
210        case ModelType.TimeSeriesPrognosis:
211          return CombineDictionaries(RegressionModelingResults, TimeSeriesPrognosisModelingResults);
212        default:
213          throw new ArgumentException("Modeling result mapping for ModelType " + modelType + " not defined.");
214      }
215    }
216
217    public static Func<double[,], double> GetModelingResultCalculator(ModelingResult modelingResult) {
218      if (RegressionModelingResults.ContainsKey(modelingResult))
219        return RegressionModelingResults[modelingResult];
220      else if (ClassificationModelingResults.ContainsKey(modelingResult))
221        return ClassificationModelingResults[modelingResult];
222      else if (TimeSeriesPrognosisModelingResults.ContainsKey(modelingResult))
223        return TimeSeriesPrognosisModelingResults[modelingResult];
224      else
225        throw new ArgumentException("Calculator for modeling result " + modelingResult + " not defined.");
226    }
227
228    public static IOperator CreateModelingResultEvaluator(ModelingResult modelingResult) {
229      IOperator opTemplate = null;
230      if (RegressionModelingResultEvaluators.ContainsKey(modelingResult))
231        opTemplate = RegressionModelingResultEvaluators[modelingResult];
232      else if (ClassificationModelingResultEvaluators.ContainsKey(modelingResult))
233        opTemplate = ClassificationModelingResultEvaluators[modelingResult];
234      else if (TimeSeriesPrognosisModelingResultEvaluators.ContainsKey(modelingResult))
235        opTemplate = TimeSeriesPrognosisModelingResultEvaluators[modelingResult];
236      else
237        throw new ArgumentException("Evaluator for modeling result " + modelingResult + " not defined.");
238      return (IOperator)opTemplate.Clone();
239    }
240
241    private static IOperator CreateEvaluator(Type evaluatorType, ModelingResult result) {
242      SimpleEvaluatorBase evaluator = (SimpleEvaluatorBase)Activator.CreateInstance(evaluatorType);
243      evaluator.GetVariableInfo("Values").ActualName = GetDatasetPart(result) + "Values";
244      evaluator.GetVariableInfo(evaluator.OutputVariableName).ActualName = result.ToString();
245      return evaluator;
246    }
247
248    private static DatasetPart GetDatasetPart(ModelingResult result) {
249      if (result.ToString().StartsWith("Training")) return DatasetPart.Training;
250      else if (result.ToString().StartsWith("Validation")) return DatasetPart.Validation;
251      else if (result.ToString().StartsWith("Test")) return DatasetPart.Test;
252      else throw new ArgumentException("Can't determine dataset part of modeling result " + result + ".");
253    }
254
255    private static Dictionary<T1, T2> CombineDictionaries<T1, T2>(
256      Dictionary<T1, T2> x,
257      Dictionary<T1, T2> y) {
258      Dictionary<T1, T2> result = new Dictionary<T1, T2>(x);
259      return x.Union(y).ToDictionary<KeyValuePair<T1, T2>, T1, T2>(p => p.Key, p => p.Value);
260    }
261  }
262}
Note: See TracBrowser for help on using the repository browser.