Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Modeling/3.2/ModelingResultCalculators.cs @ 2549

Last change on this file since 2549 was 2549, checked in by mkommend, 14 years ago

corrected typos in ModelingResults PearsonCorrelationCoefficient (ticket #782)
File size: 16.4 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2008 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Text;
26using HeuristicLab.Core;
27
28namespace HeuristicLab.Modeling {
29  public abstract class ModelingResultCalculators {
30    private enum DatasetPart { Training, Validation, Test };
31
32    private static readonly Dictionary<ModelingResult, Func<double[,], double>> ClassificationModelingResults;
33    private static readonly Dictionary<ModelingResult, Func<double[,], double>> RegressionModelingResults;
34    private static readonly Dictionary<ModelingResult, Func<double[,], double>> TimeSeriesPrognosisModelingResults;
35    private static readonly Dictionary<ModelingResult, IOperator> ClassificationModelingResultEvaluators;
36    private static readonly Dictionary<ModelingResult, IOperator> RegressionModelingResultEvaluators;
37    private static readonly Dictionary<ModelingResult, IOperator> TimeSeriesPrognosisModelingResultEvaluators;
38
39    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> regressionResults =
40      new Dictionary<Type, IEnumerable<ModelingResult>>() {
41        { typeof(SimpleMSEEvaluator),
42          new ModelingResult[] {
43            ModelingResult.TrainingMeanSquaredError,
44            ModelingResult.ValidationMeanSquaredError,
45            ModelingResult.TestMeanSquaredError
46          }},
47        { typeof(SimpleNMSEEvaluator),
48          new ModelingResult[] {
49            ModelingResult.TrainingNormalizedMeanSquaredError,
50            ModelingResult.ValidationNormalizedMeanSquaredError,
51            ModelingResult.TestNormalizedMeanSquaredError
52          }
53        },
54        { typeof(SimpleR2Evaluator),
55          new ModelingResult[] {
56            ModelingResult.TrainingCoefficientOfDetermination,
57            ModelingResult.ValidationCoefficientOfDetermination,
58            ModelingResult.TestCoefficientOfDetermination
59          }
60        },
61        { typeof(SimplePearsonCorrelationCoefficientEvaluator),
62          new ModelingResult[] {
63            ModelingResult.TrainingPearsonCorrelationCoefficient,
64            ModelingResult.ValidationPearsonCorrelationCoefficient,
65            ModelingResult.TestPearsonCorrelationCoefficient
66          }
67        },
68        { typeof(SimpleStableCorrelationCoefficientEvaluator),
69          new ModelingResult[] {
70            ModelingResult.TrainingStablePearsonCorrelationCoefficient,
71            ModelingResult.ValidationStablePearsonCorrelationCoefficient,
72            ModelingResult.TestStablePearsonCorrelationCoefficient
73          }
74        },
75        { typeof(SimpleSpearmansRankCorrelationCoefficientEvaluator),
76          new ModelingResult[] {
77            ModelingResult.TrainingSpearmansRankCorrelationCoefficient,
78            ModelingResult.ValidationSpearmansRankCorrelationCoefficient,
79            ModelingResult.TestSpearmansRankCorrelationCoefficient
80          }
81        },
82        { typeof(SimpleVarianceAccountedForEvaluator),
83          new ModelingResult[] {
84            ModelingResult.TrainingVarianceAccountedFor,
85            ModelingResult.ValidationVarianceAccountedFor,
86            ModelingResult.TestVarianceAccountedFor
87          }
88        },
89        { typeof(SimpleMeanAbsolutePercentageErrorEvaluator),
90          new ModelingResult[] {
91            ModelingResult.TrainingMeanAbsolutePercentageError,
92            ModelingResult.ValidationMeanAbsolutePercentageError,
93            ModelingResult.TestMeanAbsolutePercentageError
94          }
95        },
96        { typeof(SimpleMeanAbsolutePercentageOfRangeErrorEvaluator),
97          new ModelingResult[] {
98            ModelingResult.TrainingMeanAbsolutePercentageOfRangeError,
99            ModelingResult.ValidationMeanAbsolutePercentageOfRangeError,
100            ModelingResult.TestMeanAbsolutePercentageOfRangeError
101          }
102        }
103      };
104
105    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> timeSeriesResults =
106      new Dictionary<Type, IEnumerable<ModelingResult>>() {
107        { typeof(SimpleTheilInequalityCoefficientEvaluator),
108          new ModelingResult[] {
109            ModelingResult.TrainingTheilInequality,
110            ModelingResult.ValidationTheilInequality,
111            ModelingResult.TestTheilInequality
112          }
113        },
114        { typeof(SimpleDirectionalSymmetryEvaluator),
115          new ModelingResult[] {
116            ModelingResult.TrainingDirectionalSymmetry,
117            ModelingResult.ValidationDirectionalSymmetry,
118            ModelingResult.TestDirectionalSymmetry
119          }
120        },
121        { typeof(SimpleWeightedDirectionalSymmetryEvaluator),
122          new ModelingResult[] {
123            ModelingResult.TrainingWeightedDirectionalSymmetry,
124            ModelingResult.ValidationWeightedDirectionalSymmetry,
125            ModelingResult.TestWeightedDirectionalSymmetry
126          }
127        }
128      };
129
130    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> classificationResults =
131      new Dictionary<Type, IEnumerable<ModelingResult>>() {
132        { typeof(SimpleAccuracyEvaluator),
133          new ModelingResult[] {
134            ModelingResult.TrainingAccuracy,
135            ModelingResult.ValidationAccuracy,
136            ModelingResult.TestAccuracy
137          }
138        }
139      };
140
141
142    static ModelingResultCalculators() {
143      RegressionModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
144      ClassificationModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
145      TimeSeriesPrognosisModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
146
147      //Mean squared errors
148      RegressionModelingResults[ModelingResult.TrainingMeanSquaredError] = SimpleMSEEvaluator.Calculate;
149      RegressionModelingResults[ModelingResult.ValidationMeanSquaredError] = SimpleMSEEvaluator.Calculate;
150      RegressionModelingResults[ModelingResult.TestMeanSquaredError] = SimpleMSEEvaluator.Calculate;
151
152      //Normalized mean squared errors
153      RegressionModelingResults[ModelingResult.TrainingNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
154      RegressionModelingResults[ModelingResult.ValidationNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
155      RegressionModelingResults[ModelingResult.TestNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
156
157      //Mean absolute percentage error
158      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
159      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
160      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
161
162      //Mean absolute percentage of range error
163      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
164      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
165      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
166
167      //Coefficient of determination
168      RegressionModelingResults[ModelingResult.TrainingCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
169      RegressionModelingResults[ModelingResult.ValidationCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
170      RegressionModelingResults[ModelingResult.TestCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
171
172      //Pearson Correlation Coefficient
173      RegressionModelingResults[ModelingResult.TrainingPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
174      RegressionModelingResults[ModelingResult.ValidationPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
175      RegressionModelingResults[ModelingResult.TestPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
176
177      //Stable Pearson Correlation Coefficient
178      RegressionModelingResults[ModelingResult.TrainingStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
179      RegressionModelingResults[ModelingResult.ValidationStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
180      RegressionModelingResults[ModelingResult.TestStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
181
182      //Spearman's rank correlation coefficient
183      RegressionModelingResults[ModelingResult.TrainingSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
184      RegressionModelingResults[ModelingResult.ValidationSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
185      RegressionModelingResults[ModelingResult.TestSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
186
187      //Variance accounted for
188      RegressionModelingResults[ModelingResult.TrainingVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
189      RegressionModelingResults[ModelingResult.ValidationVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
190      RegressionModelingResults[ModelingResult.TestVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
191
192      //Accuracy
193      ClassificationModelingResults[ModelingResult.TrainingAccuracy] = SimpleAccuracyEvaluator.Calculate;
194      ClassificationModelingResults[ModelingResult.ValidationAccuracy] = SimpleAccuracyEvaluator.Calculate;
195      ClassificationModelingResults[ModelingResult.TestAccuracy] = SimpleAccuracyEvaluator.Calculate;
196
197      //Theil inequality
198      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
199      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
200      TimeSeriesPrognosisModelingResults[ModelingResult.TestTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
201
202      //Directional symmetry
203      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
204      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
205      TimeSeriesPrognosisModelingResults[ModelingResult.TestDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
206
207      //Weighted directional symmetry
208      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
209      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
210      TimeSeriesPrognosisModelingResults[ModelingResult.TestWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
211
212      #region result evaluators
213
214      RegressionModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
215      foreach (Type evaluatorT in regressionResults.Keys) {
216        foreach (ModelingResult r in regressionResults[evaluatorT]) {
217          RegressionModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
218        }
219      }
220
221      timeSeriesResults = CombineDictionaries(regressionResults, timeSeriesResults);
222      TimeSeriesPrognosisModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
223      foreach (Type evaluatorT in timeSeriesResults.Keys) {
224        foreach (ModelingResult r in timeSeriesResults[evaluatorT]) {
225          TimeSeriesPrognosisModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
226        }
227      }
228
229      classificationResults = CombineDictionaries(regressionResults, classificationResults);
230      ClassificationModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
231      foreach (Type evaluatorT in classificationResults.Keys) {
232        foreach (ModelingResult r in classificationResults[evaluatorT]) {
233          ClassificationModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
234        }
235      }
236
237      #endregion
238    }
239
240    public static Dictionary<ModelingResult, Func<double[,], double>> GetModelingResult(ModelType modelType) {
241      switch (modelType) {
242        case ModelType.Regression:
243          return CombineDictionaries(RegressionModelingResults, new Dictionary<ModelingResult, Func<double[,], double>>());
244        case ModelType.Classification:
245          return CombineDictionaries(RegressionModelingResults, ClassificationModelingResults);
246        case ModelType.TimeSeriesPrognosis:
247          return CombineDictionaries(RegressionModelingResults, TimeSeriesPrognosisModelingResults);
248        default:
249          throw new ArgumentException("Modeling result mapping for ModelType " + modelType + " not defined.");
250      }
251    }
252
253    public static Func<double[,], double> GetModelingResultCalculator(ModelingResult modelingResult) {
254      if (RegressionModelingResults.ContainsKey(modelingResult))
255        return RegressionModelingResults[modelingResult];
256      else if (ClassificationModelingResults.ContainsKey(modelingResult))
257        return ClassificationModelingResults[modelingResult];
258      else if (TimeSeriesPrognosisModelingResults.ContainsKey(modelingResult))
259        return TimeSeriesPrognosisModelingResults[modelingResult];
260      else
261        throw new ArgumentException("Calculator for modeling result " + modelingResult + " not defined.");
262    }
263
264    public static IOperator CreateModelingResultEvaluator(ModelingResult modelingResult) {
265      IOperator opTemplate = null;
266      if (RegressionModelingResultEvaluators.ContainsKey(modelingResult))
267        opTemplate = RegressionModelingResultEvaluators[modelingResult];
268      else if (ClassificationModelingResultEvaluators.ContainsKey(modelingResult))
269        opTemplate = ClassificationModelingResultEvaluators[modelingResult];
270      else if (TimeSeriesPrognosisModelingResultEvaluators.ContainsKey(modelingResult))
271        opTemplate = TimeSeriesPrognosisModelingResultEvaluators[modelingResult];
272      else
273        throw new ArgumentException("Evaluator for modeling result " + modelingResult + " not defined.");
274      return (IOperator)opTemplate.Clone();
275    }
276
277    private static IOperator CreateEvaluator(Type evaluatorType, ModelingResult result) {
278      SimpleEvaluatorBase evaluator = (SimpleEvaluatorBase)Activator.CreateInstance(evaluatorType);
279      evaluator.GetVariableInfo("Values").ActualName = GetDatasetPart(result) + "Values";
280      evaluator.GetVariableInfo(evaluator.OutputVariableName).ActualName = result.ToString();
281      return evaluator;
282    }
283
284    private static DatasetPart GetDatasetPart(ModelingResult result) {
285      if (result.ToString().StartsWith("Training")) return DatasetPart.Training;
286      else if (result.ToString().StartsWith("Validation")) return DatasetPart.Validation;
287      else if (result.ToString().StartsWith("Test")) return DatasetPart.Test;
288      else throw new ArgumentException("Can't determine dataset part of modeling result " + result + ".");
289    }
290
291    private static Dictionary<T1, T2> CombineDictionaries<T1, T2>(
292      Dictionary<T1, T2> x,
293      Dictionary<T1, T2> y) {
294      Dictionary<T1, T2> result = new Dictionary<T1, T2>(x);
295      return x.Union(y).ToDictionary<KeyValuePair<T1, T2>, T1, T2>(p => p.Key, p => p.Value);
296    }
297  }
298}
Note: See TracBrowser for help on using the repository browser.