Changeset 2379 for trunk

Timestamp:

09/23/09 13:35:55 (15 years ago)

Author:

gkronber

Message:

Implemented additional model quality metrics. #761

Location:

trunk/sources/HeuristicLab.Modeling/3.2

Files:

• DefaultRegressionOperators.cs (modified) (3 diffs)
• DefaultTimeSeriesOperators.cs (modified) (3 diffs)
• HeuristicLab.Modeling-3.2.csproj (modified) (1 diff)
• MatrixCreator.cs (modified) (2 diffs)
• ModelingResult.cs (modified) (2 diffs)
• SimpleDirectionalSymmetryEvaluator.cs (added)
• SimpleNMSEEvaluator.cs (added)
• SimpleWeightedDirectionalSymmetryEvaluator.cs (added)
• VariableEvaluationImpactCalculator.cs (modified) (1 diff)
• VariableQualityImpactCalculator.cs (modified) (1 diff)

trunk/sources/HeuristicLab.Modeling/3.2/DefaultRegressionOperators.cs

@@ -60 +60 @@
       testMseEvaluator.GetVariableInfo("MSE").ActualName = ModelingResult.TestMeanSquaredError.ToString();
       testMseEvaluator.GetVariableInfo("Values").ActualName = "TestValues";
+      #endregion
+      #region NMSE
+      SimpleNMSEEvaluator trainingNmseEvaluator = new SimpleNMSEEvaluator();
+      trainingNmseEvaluator.Name = "TrainingNmseEvaluator";
+      trainingNmseEvaluator.GetVariableInfo("NMSE").ActualName = ModelingResult.TrainingNormalizedMeanSquaredError.ToString();
+      trainingNmseEvaluator.GetVariableInfo("Values").ActualName = "TrainingValues";
+      SimpleNMSEEvaluator validationNmseEvaluator = new SimpleNMSEEvaluator();
+      validationNmseEvaluator.Name = "ValidationMseEvaluator";
+      validationNmseEvaluator.GetVariableInfo("NMSE").ActualName = ModelingResult.ValidationNormalizedMeanSquaredError.ToString();
+      validationNmseEvaluator.GetVariableInfo("Values").ActualName = "ValidationValues";
+      SimpleNMSEEvaluator testNmseEvaluator = new SimpleNMSEEvaluator();
+      testNmseEvaluator.Name = "TestNmseEvaluator";
+      testNmseEvaluator.GetVariableInfo("NMSE").ActualName = ModelingResult.TestNormalizedMeanSquaredError.ToString();
+      testNmseEvaluator.GetVariableInfo("Values").ActualName = "TestValues";
       #endregion
       #region MAPE
@@ -121 +135 @@
       seq.AddSubOperator(validationMseEvaluator);
       seq.AddSubOperator(testMseEvaluator);
+      seq.AddSubOperator(trainingNmseEvaluator);
+      seq.AddSubOperator(validationNmseEvaluator);
+      seq.AddSubOperator(testNmseEvaluator);
       seq.AddSubOperator(trainingMapeEvaluator);
       seq.AddSubOperator(validationMapeEvaluator);
@@ -167 +184 @@
       model.ExtractResult(modelScope, ModelingResult.ValidationMeanSquaredError);
       model.ExtractResult(modelScope, ModelingResult.TestMeanSquaredError);
+      model.ExtractResult(modelScope, ModelingResult.TrainingNormalizedMeanSquaredError);
+      model.ExtractResult(modelScope, ModelingResult.ValidationNormalizedMeanSquaredError);
+      model.ExtractResult(modelScope, ModelingResult.TestNormalizedMeanSquaredError);
       model.ExtractResult(modelScope, ModelingResult.TrainingMeanAbsolutePercentageError);
       model.ExtractResult(modelScope, ModelingResult.ValidationMeanAbsolutePercentageError);

trunk/sources/HeuristicLab.Modeling/3.2/DefaultTimeSeriesOperators.cs

@@ -45 +45 @@
       seq.AddSubOperator(DefaultRegressionOperators.CreatePostProcessingOperator());
 
+      #region theil inequality
       SimpleTheilInequalityCoefficientEvaluator trainingTheil = new SimpleTheilInequalityCoefficientEvaluator();
       trainingTheil.Name = "TrainingTheilInequalityEvaluator";
@@ -61 +62 @@
       seq.AddSubOperator(validationTheil);
       seq.AddSubOperator(testTheil);
+      #endregion
+
+      #region directional symmetry
+      SimpleDirectionalSymmetryEvaluator trainingDS = new SimpleDirectionalSymmetryEvaluator();
+      trainingDS.Name = "TrainingDirectionalSymmetryEvaluator";
+      trainingDS.GetVariableInfo("Values").ActualName = "TrainingValues";
+      trainingDS.GetVariableInfo("DirectionalSymmetry").ActualName = ModelingResult.TrainingDirectionalSymmetry.ToString();
+      SimpleDirectionalSymmetryEvaluator validationDS = new SimpleDirectionalSymmetryEvaluator();
+      validationDS.Name = "ValidationDirectionalSymmetryEvaluator";
+      validationDS.GetVariableInfo("Values").ActualName = "ValidationValues";
+      validationDS.GetVariableInfo("DirectionalSymmetry").ActualName = ModelingResult.ValidationDirectionalSymmetry.ToString();
+      SimpleDirectionalSymmetryEvaluator testDS = new SimpleDirectionalSymmetryEvaluator();
+      testDS.Name = "TestDirectionalSymmetryEvaluator";
+      testDS.GetVariableInfo("Values").ActualName = "TestValues";
+      testDS.GetVariableInfo("DirectionalSymmetry").ActualName = ModelingResult.TestDirectionalSymmetry.ToString();
+
+      seq.AddSubOperator(trainingDS);
+      seq.AddSubOperator(validationDS);
+      seq.AddSubOperator(testDS);
+      #endregion
+
+      #region weighted directional symmetry
+      SimpleWeightedDirectionalSymmetryEvaluator trainingWDS = new SimpleWeightedDirectionalSymmetryEvaluator();
+      trainingWDS.Name = "TrainingWeightedDirectionalSymmetryEvaluator";
+      trainingWDS.GetVariableInfo("Values").ActualName = "TrainingValues";
+      trainingWDS.GetVariableInfo("WeightedDirectionalSymmetry").ActualName = ModelingResult.TrainingWeightedDirectionalSymmetry.ToString();
+      SimpleWeightedDirectionalSymmetryEvaluator validationWDS = new SimpleWeightedDirectionalSymmetryEvaluator();
+      validationWDS.Name = "ValidationWeightedDirectionalSymmetryEvaluator";
+      validationWDS.GetVariableInfo("Values").ActualName = "ValidationValues";
+      validationWDS.GetVariableInfo("WeightedDirectionalSymmetry").ActualName = ModelingResult.ValidationWeightedDirectionalSymmetry.ToString();
+      SimpleWeightedDirectionalSymmetryEvaluator testWDS = new SimpleWeightedDirectionalSymmetryEvaluator();
+      testWDS.Name = "TestWeightedDirectionalSymmetryEvaluator";
+      testWDS.GetVariableInfo("Values").ActualName = "TestValues";
+      testWDS.GetVariableInfo("WeightedDirectionalSymmetry").ActualName = ModelingResult.TestWeightedDirectionalSymmetry.ToString();
+
+      seq.AddSubOperator(trainingWDS);
+      seq.AddSubOperator(validationWDS);
+      seq.AddSubOperator(testWDS);
+      #endregion
 
       op.OperatorGraph.AddOperator(seq);
@@ -72 +112 @@
       model.ExtractResult(modelScope, ModelingResult.ValidationTheilInequality);
       model.ExtractResult(modelScope, ModelingResult.TestTheilInequality);
+      model.ExtractResult(modelScope, ModelingResult.TrainingDirectionalSymmetry);
+      model.ExtractResult(modelScope, ModelingResult.ValidationDirectionalSymmetry);
+      model.ExtractResult(modelScope, ModelingResult.TestDirectionalSymmetry);
+      model.ExtractResult(modelScope, ModelingResult.TrainingWeightedDirectionalSymmetry);
+      model.ExtractResult(modelScope, ModelingResult.ValidationWeightedDirectionalSymmetry);
+      model.ExtractResult(modelScope, ModelingResult.TestWeightedDirectionalSymmetry);
       model.Type = ModelType.TimeSeriesPrognosis;
       return model;

trunk/sources/HeuristicLab.Modeling/3.2/HeuristicLab.Modeling-3.2.csproj

@@ -84 +84 @@
     <Compile Include="AnalyzerModel.cs" />
     <Compile Include="BestSolutionStorer.cs" />
+    <Compile Include="SimpleWeightedDirectionalSymmetryEvaluator.cs" />
+    <Compile Include="SimpleDirectionalSymmetryEvaluator.cs" />
+    <Compile Include="SimpleNMSEEvaluator.cs" />
     <Compile Include="ModelingResult.cs" />
     <Compile Include="ModelType.cs" />

trunk/sources/HeuristicLab.Modeling/3.2/MatrixCreator.cs

@@ -25 +25 @@
 
 namespace HeuristicLab.Modeling {
-  public class MatrixCreator<T> {
-    public static T[,] CreateMatrix(T[] first, T[] second) {
+  public class Matrix<T> {
+    public static T[,] Create(T[] first, T[] second) {
       if (first.Length != second.Length)
         throw new ArgumentException("Both arrays must be of same length.");
-      T[,] array = new T[first.Length,2];
+      T[,] array = new T[first.Length, 2];
       for (int i = 0; i < first.Length; i++) {
         array[i, 0] = first[i];
@@ -36 +36 @@
       return array;
     }
+
+    public static T[] GetColumn(T[,] matrix, int column) {
+      if (column >= matrix.GetLength(1)) throw new ArgumentException("Invalid column index: " + column + ". Matrix has only " + matrix.GetLength(1) + " columns.");
+      T[] result = new T[matrix.GetLength(0)];
+      for (int i = 0; i < result.Length; i++) {
+        result[i] = matrix[i, column];
+      }
+      return result;
+    }
   }
 }

trunk/sources/HeuristicLab.Modeling/3.2/ModelingResult.cs

@@ -27 +27 @@
     TestMeanSquaredError,
 
+    TrainingNormalizedMeanSquaredError,
+    ValidationNormalizedMeanSquaredError,
+    TestNormalizedMeanSquaredError,
+
     TrainingMeanAbsolutePercentageError,
     ValidationMeanAbsolutePercentageError,
@@ -51 +55 @@
     TestTheilInequality,
 
+    TrainingDirectionalSymmetry,
+    ValidationDirectionalSymmetry,
+    TestDirectionalSymmetry,
+
+    TrainingWeightedDirectionalSymmetry,
+    ValidationWeightedDirectionalSymmetry,
+    TestWeightedDirectionalSymmetry,
+
     VariableQualityImpact,
     VariableEvaluationImpact

trunk/sources/HeuristicLab.Modeling/3.2/VariableEvaluationImpactCalculator.cs

@@ -114 +114 @@
     private static double CalculateVAF(double[] referenceValues, double[] newValues) {
       try {
-        return SimpleVarianceAccountedForEvaluator.Calculate(MatrixCreator<double>.CreateMatrix(referenceValues, newValues));
+        return SimpleVarianceAccountedForEvaluator.Calculate(Matrix<double>.Create(referenceValues, newValues));
       }
       catch (ArgumentException) {

trunk/sources/HeuristicLab.Modeling/3.2/VariableQualityImpactCalculator.cs

@@ -122 +122 @@
     private static double CalculateMSE(double[] referenceValues, double[] newValues) {
       try {
-        return SimpleMSEEvaluator.Calculate(MatrixCreator<double>.CreateMatrix(referenceValues, newValues));
+        return SimpleMSEEvaluator.Calculate(Matrix<double>.Create(referenceValues, newValues));
       }
       catch (ArgumentException) {