Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/sources/HeuristicLab.Modeling/3.2/ModelingResultCalculators.cs @ 3410

Last change on this file since 3410 was 2551, checked in by mkommend, 15 years ago

added ResultCalculator for !PearsonS quality measures (ticket #782)
File size: 16.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2008 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Text;
26using HeuristicLab.Core;
27
28namespace HeuristicLab.Modeling {
29  public abstract class ModelingResultCalculators {
30    private enum DatasetPart { Training, Validation, Test };
31
32    private static readonly Dictionary<ModelingResult, Func<double[,], double>> ClassificationModelingResults;
33    private static readonly Dictionary<ModelingResult, Func<double[,], double>> RegressionModelingResults;
34    private static readonly Dictionary<ModelingResult, Func<double[,], double>> TimeSeriesPrognosisModelingResults;
35    private static readonly Dictionary<ModelingResult, IOperator> ClassificationModelingResultEvaluators;
36    private static readonly Dictionary<ModelingResult, IOperator> RegressionModelingResultEvaluators;
37    private static readonly Dictionary<ModelingResult, IOperator> TimeSeriesPrognosisModelingResultEvaluators;
38
39    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> regressionResults =
40      new Dictionary<Type, IEnumerable<ModelingResult>>() {
41        { typeof(SimpleMSEEvaluator),
42          new ModelingResult[] {
43            ModelingResult.TrainingMeanSquaredError,
44            ModelingResult.ValidationMeanSquaredError,
45            ModelingResult.TestMeanSquaredError
46          }},
47        { typeof(SimpleNMSEEvaluator),
48          new ModelingResult[] {
49            ModelingResult.TrainingNormalizedMeanSquaredError,
50            ModelingResult.ValidationNormalizedMeanSquaredError,
51            ModelingResult.TestNormalizedMeanSquaredError
52          }
53        },
54        { typeof(SimpleR2Evaluator),
55          new ModelingResult[] {
56            ModelingResult.TrainingCoefficientOfDetermination,
57            ModelingResult.ValidationCoefficientOfDetermination,
58            ModelingResult.TestCoefficientOfDetermination
59          }
60        },
61        { typeof(SimplePearsonCorrelationCoefficientEvaluator),
62          new ModelingResult[] {
63            ModelingResult.TrainingPearsonCorrelationCoefficient,
64            ModelingResult.TrainingPearsonsCorrelationCoefficient,
65            ModelingResult.ValidationPearsonCorrelationCoefficient,
66            ModelingResult.TestPearsonCorrelationCoefficient
67          }
68        },
69        { typeof(SimpleStableCorrelationCoefficientEvaluator),
70          new ModelingResult[] {
71            ModelingResult.TrainingStablePearsonCorrelationCoefficient,
72            ModelingResult.TrainingStablePearsonsCorrelationCoefficient,
73            ModelingResult.ValidationStablePearsonCorrelationCoefficient,
74            ModelingResult.TestStablePearsonCorrelationCoefficient
75          }
76        },
77        { typeof(SimpleSpearmansRankCorrelationCoefficientEvaluator),
78          new ModelingResult[] {
79            ModelingResult.TrainingSpearmansRankCorrelationCoefficient,
80            ModelingResult.ValidationSpearmansRankCorrelationCoefficient,
81            ModelingResult.TestSpearmansRankCorrelationCoefficient
82          }
83        },
84        { typeof(SimpleVarianceAccountedForEvaluator),
85          new ModelingResult[] {
86            ModelingResult.TrainingVarianceAccountedFor,
87            ModelingResult.ValidationVarianceAccountedFor,
88            ModelingResult.TestVarianceAccountedFor
89          }
90        },
91        { typeof(SimpleMeanAbsolutePercentageErrorEvaluator),
92          new ModelingResult[] {
93            ModelingResult.TrainingMeanAbsolutePercentageError,
94            ModelingResult.ValidationMeanAbsolutePercentageError,
95            ModelingResult.TestMeanAbsolutePercentageError
96          }
97        },
98        { typeof(SimpleMeanAbsolutePercentageOfRangeErrorEvaluator),
99          new ModelingResult[] {
100            ModelingResult.TrainingMeanAbsolutePercentageOfRangeError,
101            ModelingResult.ValidationMeanAbsolutePercentageOfRangeError,
102            ModelingResult.TestMeanAbsolutePercentageOfRangeError
103          }
104        }
105      };
106
107    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> timeSeriesResults =
108      new Dictionary<Type, IEnumerable<ModelingResult>>() {
109        { typeof(SimpleTheilInequalityCoefficientEvaluator),
110          new ModelingResult[] {
111            ModelingResult.TrainingTheilInequality,
112            ModelingResult.ValidationTheilInequality,
113            ModelingResult.TestTheilInequality
114          }
115        },
116        { typeof(SimpleDirectionalSymmetryEvaluator),
117          new ModelingResult[] {
118            ModelingResult.TrainingDirectionalSymmetry,
119            ModelingResult.ValidationDirectionalSymmetry,
120            ModelingResult.TestDirectionalSymmetry
121          }
122        },
123        { typeof(SimpleWeightedDirectionalSymmetryEvaluator),
124          new ModelingResult[] {
125            ModelingResult.TrainingWeightedDirectionalSymmetry,
126            ModelingResult.ValidationWeightedDirectionalSymmetry,
127            ModelingResult.TestWeightedDirectionalSymmetry
128          }
129        }
130      };
131
132    private static readonly Dictionary<Type, IEnumerable<ModelingResult>> classificationResults =
133      new Dictionary<Type, IEnumerable<ModelingResult>>() {
134        { typeof(SimpleAccuracyEvaluator),
135          new ModelingResult[] {
136            ModelingResult.TrainingAccuracy,
137            ModelingResult.ValidationAccuracy,
138            ModelingResult.TestAccuracy
139          }
140        }
141      };
142
143
144    static ModelingResultCalculators() {
145      RegressionModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
146      ClassificationModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
147      TimeSeriesPrognosisModelingResults = new Dictionary<ModelingResult, Func<double[,], double>>();
148
149      //Mean squared errors
150      RegressionModelingResults[ModelingResult.TrainingMeanSquaredError] = SimpleMSEEvaluator.Calculate;
151      RegressionModelingResults[ModelingResult.ValidationMeanSquaredError] = SimpleMSEEvaluator.Calculate;
152      RegressionModelingResults[ModelingResult.TestMeanSquaredError] = SimpleMSEEvaluator.Calculate;
153
154      //Normalized mean squared errors
155      RegressionModelingResults[ModelingResult.TrainingNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
156      RegressionModelingResults[ModelingResult.ValidationNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
157      RegressionModelingResults[ModelingResult.TestNormalizedMeanSquaredError] = SimpleNMSEEvaluator.Calculate;
158
159      //Mean absolute percentage error
160      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
161      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
162      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageError] = SimpleMeanAbsolutePercentageErrorEvaluator.Calculate;
163
164      //Mean absolute percentage of range error
165      RegressionModelingResults[ModelingResult.TrainingMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
166      RegressionModelingResults[ModelingResult.ValidationMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
167      RegressionModelingResults[ModelingResult.TestMeanAbsolutePercentageOfRangeError] = SimpleMeanAbsolutePercentageOfRangeErrorEvaluator.Calculate;
168
169      //Coefficient of determination
170      RegressionModelingResults[ModelingResult.TrainingCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
171      RegressionModelingResults[ModelingResult.ValidationCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
172      RegressionModelingResults[ModelingResult.TestCoefficientOfDetermination] = SimpleR2Evaluator.Calculate;
173
174      //Pearson Correlation Coefficient
175      RegressionModelingResults[ModelingResult.TrainingPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
176      RegressionModelingResults[ModelingResult.TrainingPearsonsCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
177      RegressionModelingResults[ModelingResult.ValidationPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
178      RegressionModelingResults[ModelingResult.TestPearsonCorrelationCoefficient] = SimplePearsonCorrelationCoefficientEvaluator.Calculate;
179
180      //Stable Pearson Correlation Coefficient
181      RegressionModelingResults[ModelingResult.TrainingStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
182      RegressionModelingResults[ModelingResult.TrainingStablePearsonsCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
183      RegressionModelingResults[ModelingResult.ValidationStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
184      RegressionModelingResults[ModelingResult.TestStablePearsonCorrelationCoefficient] = SimpleStableCorrelationCoefficientEvaluator.Calculate;
185
186      //Spearman's rank correlation coefficient
187      RegressionModelingResults[ModelingResult.TrainingSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
188      RegressionModelingResults[ModelingResult.ValidationSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
189      RegressionModelingResults[ModelingResult.TestSpearmansRankCorrelationCoefficient] = SimpleSpearmansRankCorrelationCoefficientEvaluator.Calculate;
190
191      //Variance accounted for
192      RegressionModelingResults[ModelingResult.TrainingVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
193      RegressionModelingResults[ModelingResult.ValidationVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
194      RegressionModelingResults[ModelingResult.TestVarianceAccountedFor] = SimpleVarianceAccountedForEvaluator.Calculate;
195
196      //Accuracy
197      ClassificationModelingResults[ModelingResult.TrainingAccuracy] = SimpleAccuracyEvaluator.Calculate;
198      ClassificationModelingResults[ModelingResult.ValidationAccuracy] = SimpleAccuracyEvaluator.Calculate;
199      ClassificationModelingResults[ModelingResult.TestAccuracy] = SimpleAccuracyEvaluator.Calculate;
200
201      //Theil inequality
202      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
203      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
204      TimeSeriesPrognosisModelingResults[ModelingResult.TestTheilInequality] = SimpleTheilInequalityCoefficientEvaluator.Calculate;
205
206      //Directional symmetry
207      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
208      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
209      TimeSeriesPrognosisModelingResults[ModelingResult.TestDirectionalSymmetry] = SimpleDirectionalSymmetryEvaluator.Calculate;
210
211      //Weighted directional symmetry
212      TimeSeriesPrognosisModelingResults[ModelingResult.TrainingWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
213      TimeSeriesPrognosisModelingResults[ModelingResult.ValidationWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
214      TimeSeriesPrognosisModelingResults[ModelingResult.TestWeightedDirectionalSymmetry] = SimpleWeightedDirectionalSymmetryEvaluator.Calculate;
215
216      #region result evaluators
217
218      RegressionModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
219      foreach (Type evaluatorT in regressionResults.Keys) {
220        foreach (ModelingResult r in regressionResults[evaluatorT]) {
221          RegressionModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
222        }
223      }
224
225      timeSeriesResults = CombineDictionaries(regressionResults, timeSeriesResults);
226      TimeSeriesPrognosisModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
227      foreach (Type evaluatorT in timeSeriesResults.Keys) {
228        foreach (ModelingResult r in timeSeriesResults[evaluatorT]) {
229          TimeSeriesPrognosisModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
230        }
231      }
232
233      classificationResults = CombineDictionaries(regressionResults, classificationResults);
234      ClassificationModelingResultEvaluators = new Dictionary<ModelingResult, IOperator>();
235      foreach (Type evaluatorT in classificationResults.Keys) {
236        foreach (ModelingResult r in classificationResults[evaluatorT]) {
237          ClassificationModelingResultEvaluators[r] = CreateEvaluator(evaluatorT, r);
238        }
239      }
240
241      #endregion
242    }
243
244    public static Dictionary<ModelingResult, Func<double[,], double>> GetModelingResult(ModelType modelType) {
245      switch (modelType) {
246        case ModelType.Regression:
247          return CombineDictionaries(RegressionModelingResults, new Dictionary<ModelingResult, Func<double[,], double>>());
248        case ModelType.Classification:
249          return CombineDictionaries(RegressionModelingResults, ClassificationModelingResults);
250        case ModelType.TimeSeriesPrognosis:
251          return CombineDictionaries(RegressionModelingResults, TimeSeriesPrognosisModelingResults);
252        default:
253          throw new ArgumentException("Modeling result mapping for ModelType " + modelType + " not defined.");
254      }
255    }
256
257    public static Func<double[,], double> GetModelingResultCalculator(ModelingResult modelingResult) {
258      if (RegressionModelingResults.ContainsKey(modelingResult))
259        return RegressionModelingResults[modelingResult];
260      else if (ClassificationModelingResults.ContainsKey(modelingResult))
261        return ClassificationModelingResults[modelingResult];
262      else if (TimeSeriesPrognosisModelingResults.ContainsKey(modelingResult))
263        return TimeSeriesPrognosisModelingResults[modelingResult];
264      else
265        throw new ArgumentException("Calculator for modeling result " + modelingResult + " not defined.");
266    }
267
268    public static IOperator CreateModelingResultEvaluator(ModelingResult modelingResult) {
269      IOperator opTemplate = null;
270      if (RegressionModelingResultEvaluators.ContainsKey(modelingResult))
271        opTemplate = RegressionModelingResultEvaluators[modelingResult];
272      else if (ClassificationModelingResultEvaluators.ContainsKey(modelingResult))
273        opTemplate = ClassificationModelingResultEvaluators[modelingResult];
274      else if (TimeSeriesPrognosisModelingResultEvaluators.ContainsKey(modelingResult))
275        opTemplate = TimeSeriesPrognosisModelingResultEvaluators[modelingResult];
276      else
277        throw new ArgumentException("Evaluator for modeling result " + modelingResult + " not defined.");
278      return (IOperator)opTemplate.Clone();
279    }
280
281    private static IOperator CreateEvaluator(Type evaluatorType, ModelingResult result) {
282      SimpleEvaluatorBase evaluator = (SimpleEvaluatorBase)Activator.CreateInstance(evaluatorType);
283      evaluator.GetVariableInfo("Values").ActualName = GetDatasetPart(result) + "Values";
284      evaluator.GetVariableInfo(evaluator.OutputVariableName).ActualName = result.ToString();
285      return evaluator;
286    }
287
288    private static DatasetPart GetDatasetPart(ModelingResult result) {
289      if (result.ToString().StartsWith("Training")) return DatasetPart.Training;
290      else if (result.ToString().StartsWith("Validation")) return DatasetPart.Validation;
291      else if (result.ToString().StartsWith("Test")) return DatasetPart.Test;
292      else throw new ArgumentException("Can't determine dataset part of modeling result " + result + ".");
293    }
294
295    private static Dictionary<T1, T2> CombineDictionaries<T1, T2>(
296      Dictionary<T1, T2> x,
297      Dictionary<T1, T2> y) {
298      Dictionary<T1, T2> result = new Dictionary<T1, T2>(x);
299      return x.Union(y).ToDictionary<KeyValuePair<T1, T2>, T1, T2>(p => p.Key, p => p.Value);
300    }
301  }
302}
Note: See TracBrowser for help on using the repository browser.