Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Modeling/3.2/ModelingResultCalculators.cs @ 2428

Last change on this file since 2428 was 2428, checked in by gkronber, 15 years ago

Implemented #782 (Additional model quality metrics: Pearson product-moment correlation coefficient and Spearman's rank correlation coefficient)
File size: 16.1 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2008 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Text;
26using HeuristicLab.Core;
27
28namespace HeuristicLab.Modeling {
29  public abstract class ModelingResultCalculators {
30    private enum DatasetPart { Training, Validation, Test };
31
32    private static readonly Dictionary<ModelingResult, Func<double[,], double>> ClassificationModelingResults;
33    private static readonly Dictionary<ModelingResult, Func<double[,], double>> RegressionModelingResults;
34    private static readonly Dictionary<ModelingResult, Func<double[,], double>> TimeSeriesPrognosisModelingResults;
35    private static readonly Dictionary<ModelingResult, IOperator> ClassificationModelingResultEvaluators;
36    private static readonly Dictionary<ModelingResult, IOperator> RegressionModelingResultEvaluators;
37    private static readonly Dictionary<ModelingResult, IOperator> TimeSeriesPrognosisModelingResultEvaluators;
38
39    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> regressionResults =
40      new Dictionary<Type, IEnumerable<ModelingResult>>() {
41        { typeof(SimpleMSEEvaluator),
42          new ModelingResult[] {
43            ModelingResult.TrainingMeanSquaredError,
44            ModelingResult.ValidationMeanSquaredError,
45            ModelingResult.TestMeanSquaredError
46          }},
47        { typeof(SimpleNMSEEvaluator),
48          new ModelingResult[] {
49            ModelingResult.TrainingNormalizedMeanSquaredError,
50            ModelingResult.ValidationNormalizedMeanSquaredError,
51            ModelingResult.TestNormalizedMeanSquaredError
52          }
53        },
54        { typeof(SimpleR2Evaluator),
55          new ModelingResult[] {
56            ModelingResult.TrainingCoefficientOfDetermination,
57            ModelingResult.ValidationCoefficientOfDetermination,
58            ModelingResult.TestCoefficientOfDetermination
59          }
60        },
61        { typeof(SimplePearsonCorrelationCoefficientEvaluator),
62          new ModelingResult[] {
63            ModelingResult.TrainingPearsonsCorrelationCoefficient,
64            ModelingResult.ValidationPearsonCorrlationCoefficent,
65            ModelingResult.TestPearsonCorrelationCoefficient
66          }
67        },
68        { typeof(SimpleSpearmansRankCorrelationCoefficientEvaluator),
69          new ModelingResult[] {
70            ModelingResult.TrainingSpearmansRankCorrelationCoefficient,
71            ModelingResult.ValidationSpearmansRankCorrelationCoefficient,
72            ModelingResult.TestSpearmansRankCorrelationCoefficient
73          }
74        },
75        { typeof(SimpleVarianceAccountedForEvaluator),
76          new ModelingResult[] {
77            ModelingResult.TrainingVarianceAccountedFor,
78            ModelingResult.ValidationVarianceAccountedFor,
79            ModelingResult.TestVarianceAccountedFor
80          }
81        },
82        { typeof(SimpleMeanAbsolutePercentageErrorEvaluator),
83          new ModelingResult[] {
84            ModelingResult.TrainingMeanAbsolutePercentageError,
85            ModelingResult.ValidationMeanAbsolutePercentageError,
86            ModelingResult.TestMeanAbsolutePercentageError
87          }
88        },
89        { typeof(SimpleMeanAbsolutePercentageOfRangeErrorEvaluator),
90          new ModelingResult[] {
91            ModelingResult.TrainingMeanAbsolutePercentageOfRangeError,
92            ModelingResult.ValidationMeanAbsolutePercentageOfRangeError,
93            ModelingResult.TestMeanAbsolutePercentageOfRangeError
94          }
95        }
96      };
97
98    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> timeSeriesResults =
99      new Dictionary<Type, IEnumerable<ModelingResult>>() {
100        { typeof(SimpleTheilInequalityCoefficientEvaluator),
101          new ModelingResult[] {
102            ModelingResult.TrainingTheilInequality,
103            ModelingResult.ValidationTheilInequality,
104            ModelingResult.TestTheilInequality
105          }
106        },
107        { typeof(SimpleDirectionalSymmetryEvaluator),
108          new ModelingResult[] {
109            ModelingResult.TrainingDirectionalSymmetry,
110            ModelingResult.ValidationDirectionalSymmetry,
111            ModelingResult.TestDirectionalSymmetry
112          }
113        },
114        { typeof(SimpleWeightedDirectionalSymmetryEvaluator),
115          new ModelingResult[] {
116            ModelingResult.TrainingWeightedDirectionalSymmetry,
117            ModelingResult.ValidationWeightedDirectionalSymmetry,
118            ModelingResult.TestWeightedDirectionalSymmetry
119          }
120        }
121      };
122
123    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> classificationResults =
124      new Dictionary<Type, IEnumerable<ModelingResult>>() {
125        { typeof(SimpleAccuracyEvaluator),
126          new ModelingResult[] {
127            ModelingResult.TrainingAccuracy,
128            ModelingResult.ValidationAccuracy,
129            ModelingResult.TestAccuracy
130          }
131        }
132      };
133
134
135    static ModelingResultCalculators() {
136      RegressionModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
137      ClassificationModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
138      TimeSeriesPrognosisModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
139
140      //Mean squared errors
141      RegressionModelingResults[ModelingResult.TrainingMeanSquaredError] = SimpleMSEEvaluator.Calculate;
142      RegressionModelingResults[ModelingResult.ValidationMeanSquaredError] = SimpleMSEEvaluator.Calculate;
143      RegressionModelingResults[ModelingResult.TestMeanSquaredError] = SimpleMSEEvaluator.Calculate;
144
145      //Normalized mean squared errors
146      RegressionModelingResults[ModelingResult.TrainingNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
147      RegressionModelingResults[ModelingResult.ValidationNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
148      RegressionModelingResults[ModelingResult.TestNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
149
150      //Mean absolute percentage error
151      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
152      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
153      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
154
155      //Mean absolute percentage of range error
156      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
157      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
158      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
159
160      //Coefficient of determination
161      RegressionModelingResults[ModelingResult.TrainingCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
162      RegressionModelingResults[ModelingResult.ValidationCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
163      RegressionModelingResults[ModelingResult.TestCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
164
165      //Pearson Correlation Coefficient
166      RegressionModelingResults[ModelingResult.TrainingPearsonsCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
167      RegressionModelingResults[ModelingResult.ValidationPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
168      RegressionModelingResults[ModelingResult.TestPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
169
170      //Stable Pearson Correlation Coefficient
171      RegressionModelingResults[ModelingResult.TrainingStablePearsonsCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
172      RegressionModelingResults[ModelingResult.ValidationStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
173      RegressionModelingResults[ModelingResult.TestStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
174
175      //Spearman's rank correlation coefficient
176      RegressionModelingResults[ModelingResult.TrainingSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
177      RegressionModelingResults[ModelingResult.ValidationSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
178      RegressionModelingResults[ModelingResult.TestSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
179
180      //Variance accounted for
181      RegressionModelingResults[ModelingResult.TrainingVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
182      RegressionModelingResults[ModelingResult.ValidationVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
183      RegressionModelingResults[ModelingResult.TestVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
184
185      //Accuracy
186      ClassificationModelingResults[ModelingResult.TrainingAccuracy] = SimpleAccuracyEvaluator.Calculate;
187      ClassificationModelingResults[ModelingResult.ValidationAccuracy] = SimpleAccuracyEvaluator.Calculate;
188      ClassificationModelingResults[ModelingResult.TestAccuracy] = SimpleAccuracyEvaluator.Calculate;
189
190      //Theil inequality
191      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
192      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
193      TimeSeriesPrognosisModelingResults[ModelingResult.TestTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
194
195      //Directional symmetry
196      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
197      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
198      TimeSeriesPrognosisModelingResults[ModelingResult.TestDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
199
200      //Weighted directional symmetry
201      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
202      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
203      TimeSeriesPrognosisModelingResults[ModelingResult.TestWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
204
205      #region result evaluators
206
207      RegressionModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
208      foreach (Type evaluatorT in regressionResults.Keys) {
209        foreach (ModelingResult r in regressionResults[evaluatorT]) {
210          RegressionModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
211        }
212      }
213
214      timeSeriesResults = CombineDictionaries(regressionResults, timeSeriesResults);
215      TimeSeriesPrognosisModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
216      foreach (Type evaluatorT in timeSeriesResults.Keys) {
217        foreach (ModelingResult r in timeSeriesResults[evaluatorT]) {
218          TimeSeriesPrognosisModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
219        }
220      }
221
222      classificationResults = CombineDictionaries(regressionResults, classificationResults);
223      ClassificationModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
224      foreach (Type evaluatorT in classificationResults.Keys) {
225        foreach (ModelingResult r in classificationResults[evaluatorT]) {
226          ClassificationModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
227        }
228      }
229
230      #endregion
231    }
232
233    public static Dictionary<ModelingResult, Func<double[,], double>> GetModelingResult(ModelType modelType) {
234      switch (modelType) {
235        case ModelType.Regression:
236          return CombineDictionaries(RegressionModelingResults, new Dictionary<ModelingResult, Func<double[,], double>>());
237        case ModelType.Classification:
238          return CombineDictionaries(RegressionModelingResults, ClassificationModelingResults);
239        case ModelType.TimeSeriesPrognosis:
240          return CombineDictionaries(RegressionModelingResults, TimeSeriesPrognosisModelingResults);
241        default:
242          throw new ArgumentException("Modeling result mapping for ModelType " + modelType + " not defined.");
243      }
244    }
245
246    public static Func<double[,], double> GetModelingResultCalculator(ModelingResult modelingResult) {
247      if (RegressionModelingResults.ContainsKey(modelingResult))
248        return RegressionModelingResults[modelingResult];
249      else if (ClassificationModelingResults.ContainsKey(modelingResult))
250        return ClassificationModelingResults[modelingResult];
251      else if (TimeSeriesPrognosisModelingResults.ContainsKey(modelingResult))
252        return TimeSeriesPrognosisModelingResults[modelingResult];
253      else
254        throw new ArgumentException("Calculator for modeling result " + modelingResult + " not defined.");
255    }
256
257    public static IOperator CreateModelingResultEvaluator(ModelingResult modelingResult) {
258      IOperator opTemplate = null;
259      if (RegressionModelingResultEvaluators.ContainsKey(modelingResult))
260        opTemplate = RegressionModelingResultEvaluators[modelingResult];
261      else if (ClassificationModelingResultEvaluators.ContainsKey(modelingResult))
262        opTemplate = ClassificationModelingResultEvaluators[modelingResult];
263      else if (TimeSeriesPrognosisModelingResultEvaluators.ContainsKey(modelingResult))
264        opTemplate = TimeSeriesPrognosisModelingResultEvaluators[modelingResult];
265      else
266        throw new ArgumentException("Evaluator for modeling result " + modelingResult + " not defined.");
267      return (IOperator)opTemplate.Clone();
268    }
269
270    private static IOperator CreateEvaluator(Type evaluatorType, ModelingResult result) {
271      SimpleEvaluatorBase evaluator = (SimpleEvaluatorBase)Activator.CreateInstance(evaluatorType);
272      evaluator.GetVariableInfo("Values").ActualName = GetDatasetPart(result) + "Values";
273      evaluator.GetVariableInfo(evaluator.OutputVariableName).ActualName = result.ToString();
274      return evaluator;
275    }
276
277    private static DatasetPart GetDatasetPart(ModelingResult result) {
278      if (result.ToString().StartsWith("Training")) return DatasetPart.Training;
279      else if (result.ToString().StartsWith("Validation")) return DatasetPart.Validation;
280      else if (result.ToString().StartsWith("Test")) return DatasetPart.Test;
281      else throw new ArgumentException("Can't determine dataset part of modeling result " + result + ".");
282    }
283
284    private static Dictionary<T1, T2> CombineDictionaries<T1, T2>(
285      Dictionary<T1, T2> x,
286      Dictionary<T1, T2> y) {
287      Dictionary<T1, T2> result = new Dictionary<T1, T2>(x);
288      return x.Union(y).ToDictionary<KeyValuePair<T1, T2>, T1, T2>(p => p.Key, p => p.Value);
289    }
290  }
291}
Note: See TracBrowser for help on using the repository browser.