Changeset 15450 for branches/MathNetNumerics-Exploration-2789/HeuristicLab.Algorithms.DataAnalysis.Experimental/Splines.cs

Timestamp:

11/06/17 13:12:41 (6 years ago)

Author:

gkronber

Message:

#2789 more tests with CV and automatic determination of smoothing parameter

File:

• branches/MathNetNumerics-Exploration-2789/HeuristicLab.Algorithms.DataAnalysis.Experimental/Splines.cs (modified) (7 diffs)

branches/MathNetNumerics-Exploration-2789/HeuristicLab.Algorithms.DataAnalysis.Experimental/Splines.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
@@ -156 +157 @@
           case "CUBGCV": {
               CubicSplineGCV.CubGcvReport report;
-              var model = 
-                CubicSplineGCV.CalculateCubicSpline(x, y, 
+              var model =
+                CubicSplineGCV.CalculateCubicSpline(x, y,
                 Problem.ProblemData.TargetVariable, inputVars, out report);
               var targetVar = Problem.ProblemData.TargetVariable;
@@ -165 +166 @@
               Results.Add(new Result("Estimated error variance", new DoubleValue(report.estimatedErrorVariance)));
               Results.Add(new Result("Estimated RSS degrees of freedom", new DoubleValue(report.estimatedRSSDegreesOfFreedom)));
-              Results.Add(new Result("Estimated treu mean squared error at data points", new DoubleValue(report.estimatedTrueMeanSquaredErrorAtDataPoints)));
+              Results.Add(new Result("Estimated true mean squared error at data points", new DoubleValue(report.estimatedTrueMeanSquaredErrorAtDataPoints)));
               Results.Add(new Result("Mean square of DF", new DoubleValue(report.meanSquareOfDf)));
               Results.Add(new Result("Mean square residual", new DoubleValue(report.meanSquareResiudal)));
@@ -204 +205 @@
     public static IRegressionEnsembleModel CalculatePenalizedRegressionSpline(double[] x, double[] y, double lambda, string targetVar, string[] inputVars,
       out double avgTrainRMSE,
-      out double loocvRMSE) {
+      out double cvRMSE) {
       int info;
       alglib.spline1dinterpolant interpolant;
@@ -211 +212 @@
       n = (int)Math.Max(50, 3 * Math.Sqrt(n));
 
-      double[] x_loo = new double[x.Length - 1];
-      double[] y_loo = new double[y.Length - 1];
+      int folds = 10;
+      int foldSize = x.Length <= 30 ? 1 : x.Length / folds;
+      if (foldSize == 1) folds = x.Length;
+
+      double[] x_loo = new double[(folds - 1) * foldSize];
+      double[] y_loo = new double[(folds - 1) * foldSize];
 
       var models = new List<IRegressionModel>();
-      var sumTrainRmse = 0.0;
-      var sse = 0.0;
-
-      for (int i = 0; i < x.Length; i++) {
-        Array.Copy(x, 0, x_loo, 0, i);
-        Array.Copy(x, i + 1, x_loo, i, x.Length - (i + 1));
-        Array.Copy(y, 0, y_loo, 0, i);
-        Array.Copy(y, i + 1, y_loo, i, y.Length - (i + 1));
+      var sumTrainSE = 0.0;
+      int nTrain = 0;
+      var sumTestSE = 0.0;
+      int nTest = 0;
+
+
+      for (int i = 0; i < folds; i++) {
+        if (i < folds - 1) {
+          //Console.WriteLine("Copy from {0} to {1} n = {2}", (i + 1) * foldSize, i * foldSize, (folds - i - 1) * foldSize);
+          Array.Copy(x, (i + 1) * foldSize, x_loo, i * foldSize, (folds - i - 1) * foldSize);
+          Array.Copy(y, (i + 1) * foldSize, y_loo, i * foldSize, (folds - i - 1) * foldSize);
+        }
         alglib.spline1dfitpenalized(x_loo, y_loo, n, lambda, out info, out interpolant, out rep);
-        var y_pred = alglib.spline1dcalc(interpolant, x[i]);
-        double res = y[i] - y_pred;
-        sse += res * res;
-        sumTrainRmse += rep.rmserror;
-
-        var model = new Spline1dModel(interpolant, targetVar, inputVars);
-        models.Add(model);
-      }
-
-      loocvRMSE = Math.Sqrt(sse / x.Length);
-      avgTrainRMSE = sumTrainRmse / x.Length;
+        Debug.Assert(y_loo.All(yi => yi != 0.0));
+        Debug.Assert(x_loo.All(xi => xi != 0.0));
+
+        // training error
+        for (int j = 0; j < x_loo.Length; j++) {
+          var y_pred = alglib.spline1dcalc(interpolant, x[i]);
+          double res = y[j] - y_pred;
+          sumTrainSE += res * res;
+          nTrain++;
+        }
+        // test
+        for (int j = i * foldSize; j < (i + 1) * foldSize; j++) {
+          var y_pred = alglib.spline1dcalc(interpolant, x[i]);
+          double res = y[j] - y_pred;
+          sumTestSE += res * res;
+          nTest++;
+        }
+
+        models.Add(new Spline1dModel(interpolant, targetVar, inputVars));
+
+        if (i < folds - 1) {
+          //Console.WriteLine("Copy from {0} to {1} n = {2}", i * foldSize, i * foldSize, foldSize);
+          // add current fold for next iteration
+          Array.Copy(x, i * foldSize, x_loo, i * foldSize, foldSize);
+          Array.Copy(y, i * foldSize, y_loo, i * foldSize, foldSize);
+        }
+      }
+
+      cvRMSE = Math.Sqrt(sumTestSE / nTest);
+      avgTrainRMSE = Math.Sqrt(sumTrainSE / nTrain);
 
       return new RegressionEnsembleModel(models);
@@ -352 +380 @@
     public static IRegressionEnsembleModel CalculateSmoothingSplineReinsch(double[] x, double[] y, double tol, string targetVar, string[] inputVars,
       out double avgTrainRMSE,
-      out double loocvRMSE) {
+      out double cvRMSE) {
       int info;
 
-      double[] x_loo = new double[x.Length - 1];
-      double[] y_loo = new double[y.Length - 1];
+      int folds = 10;
+      int foldSize = x.Length <= 30 ? 1 : x.Length / folds;
+      if (foldSize == 1) folds = x.Length;
+
+      double[] x_loo = new double[(folds - 1) * foldSize];
+      double[] y_loo = new double[(folds - 1) * foldSize];
 
       var models = new List<IRegressionModel>();
-      var sumTrainRmse = 0.0;
-      var sse = 0.0;
-
-      for (int i = 0; i < x.Length; i++) {
-        Array.Copy(x, i + 1, x_loo, i, x.Length - (i + 1));
-        Array.Copy(y, i + 1, y_loo, i, y.Length - (i + 1));
-
-        var model = CalculateSmoothingSplineReinsch(x_loo, y_loo, inputVars, tol, x_loo.Length + 1, targetVar);
-
-        var y_pred = model.GetEstimatedValue(x[i]);
-        double res = y[i] - y_pred;
-        sse += res * res;
+      var sumTrainSE = 0.0;
+      int nTrain = 0;
+      var sumTestSE = 0.0;
+      int nTest = 0;
+
+
+      for (int i = 0; i < folds; i++) {
+        if (i < folds - 1) {
+          Array.Copy(x, (i + 1) * foldSize, x_loo, i * foldSize, x.Length - ((i + 1) * foldSize) - 1);
+          Array.Copy(y, (i + 1) * foldSize, y_loo, i * foldSize, y.Length - ((i + 1) * foldSize) - 1);
+        }
+        var model = CalculateSmoothingSplineReinsch(x_loo, y_loo, inputVars, tol, ((folds - 1) * foldSize) - 1, targetVar);
+
+        // training error
+        for(int j = 0;j<x_loo.Length;j++) {
+          var y_pred = model.GetEstimatedValue(x[j]);
+          double res = y[j] - y_pred;
+          sumTrainSE += res * res;
+          nTrain++;
+        }
+        // test
+        for (int j = i * foldSize; j < (i + 1) * foldSize; j++) {
+          var y_pred = model.GetEstimatedValue(x[j]);
+          double res = y[j] - y_pred;
+          sumTestSE += res * res;
+          nTest++;
+        }
 
         models.Add(model);
 
-        // add current row for next iteration
-        if (i < x_loo.Length) {
-          x_loo[i] = x[i];
-          y_loo[i] = y[i];
-        }
-      }
-
-      loocvRMSE = Math.Sqrt(sse / x.Length);
-      avgTrainRMSE = sumTrainRmse / x.Length;
+        if (i < folds - 1) {
+          // add current fold for next iteration
+          Array.Copy(x, i * foldSize, x_loo, i * foldSize, foldSize);
+          Array.Copy(y, i * foldSize, y_loo, i * foldSize, foldSize);
+        }
+      }
+
+      cvRMSE = Math.Sqrt(sumTestSE / nTest);
+      avgTrainRMSE = Math.Sqrt(sumTrainSE / nTrain);
 
       return new RegressionEnsembleModel(models);
@@ -388 +435 @@
 
 
-    // automatically determines tolerance using loo cv
+    // automatically determines tolerance using loo cv, SLOW!
     public static IRegressionModel CalculateSmoothingSplineReinsch(double[] x, double[] y, string[] inputVars, string targetVar,
-      out double optTolerance, out double looRMSE) {
+      out double optTolerance, out double cvRMSE) {
       var maxTolerance = y.StandardDeviation();
       var minTolerance = maxTolerance * 1E-6;
       optTolerance = maxTolerance;
       var tol = maxTolerance;
-      looRMSE = double.PositiveInfinity;
+      cvRMSE = double.PositiveInfinity;
       IRegressionModel bestModel = new ConstantModel(y.Average(), targetVar);
       while (tol > minTolerance) {
-        double loo_rmse;
+        double curCVRMSE;
         double avgTrainRmse;
         var model = Splines.CalculateSmoothingSplineReinsch(
-          x, y, tol, targetVar, inputVars, out avgTrainRmse, out loo_rmse);
-
-        if (loo_rmse < looRMSE) {
-          looRMSE = loo_rmse;
+          x, y, tol, targetVar, inputVars, out avgTrainRmse, out curCVRMSE);
+
+        if (curCVRMSE < cvRMSE) {
+          cvRMSE = curCVRMSE;
           optTolerance = tol;
           bestModel = model;