[14370] | 1 | #region License Information
|
---|
| 2 | /* HeuristicLab
|
---|
[17180] | 3 | * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
[14370] | 4 | *
|
---|
| 5 | * This file is part of HeuristicLab.
|
---|
| 6 | *
|
---|
| 7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
| 8 | * it under the terms of the GNU General Public License as published by
|
---|
| 9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
| 10 | * (at your option) any later version.
|
---|
| 11 | *
|
---|
| 12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
| 15 | * GNU General Public License for more details.
|
---|
| 16 | *
|
---|
| 17 | * You should have received a copy of the GNU General Public License
|
---|
| 18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
| 19 | */
|
---|
| 20 | #endregion
|
---|
| 21 |
|
---|
| 22 | using System;
|
---|
[15023] | 23 | using System.Collections.Generic;
|
---|
[13927] | 24 | using System.Linq;
|
---|
[14674] | 25 | using System.Threading;
|
---|
[13927] | 26 | using HeuristicLab.Analysis;
|
---|
| 27 | using HeuristicLab.Common;
|
---|
| 28 | using HeuristicLab.Core;
|
---|
| 29 | using HeuristicLab.Data;
|
---|
| 30 | using HeuristicLab.Optimization;
|
---|
| 31 | using HeuristicLab.Parameters;
|
---|
[16565] | 32 | using HEAL.Attic;
|
---|
[13927] | 33 | using HeuristicLab.Problems.DataAnalysis;
|
---|
[13930] | 34 | using HeuristicLab.Problems.DataAnalysis.Symbolic;
|
---|
| 35 | using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
|
---|
[17991] | 36 | using HeuristicLab.Analysis.Statistics;
|
---|
[13927] | 37 |
|
---|
[14225] | 38 | namespace HeuristicLab.Algorithms.DataAnalysis.Glmnet {
|
---|
[13927] | 39 | [Item("Elastic-net Linear Regression (LR)", "Linear regression with elastic-net regularization (wrapper for glmnet)")]
|
---|
| 40 | [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 110)]
|
---|
[16565] | 41 | [StorableType("529EDD40-91F3-4F3E-929F-852A3EF9B02B")]
|
---|
[13927] | 42 | public sealed class ElasticNetLinearRegression : FixedDataAnalysisAlgorithm<IRegressionProblem> {
|
---|
| 43 | private const string PenalityParameterName = "Penality";
|
---|
[14395] | 44 | private const string LambdaParameterName = "Lambda";
|
---|
[13927] | 45 | #region parameters
|
---|
| 46 | public IFixedValueParameter<DoubleValue> PenalityParameter {
|
---|
| 47 | get { return (IFixedValueParameter<DoubleValue>)Parameters[PenalityParameterName]; }
|
---|
| 48 | }
|
---|
[14395] | 49 | public IValueParameter<DoubleValue> LambdaParameter {
|
---|
| 50 | get { return (IValueParameter<DoubleValue>)Parameters[LambdaParameterName]; }
|
---|
[13930] | 51 | }
|
---|
[13927] | 52 | #endregion
|
---|
| 53 | #region properties
|
---|
| 54 | public double Penality {
|
---|
| 55 | get { return PenalityParameter.Value.Value; }
|
---|
| 56 | set { PenalityParameter.Value.Value = value; }
|
---|
| 57 | }
|
---|
[14395] | 58 | public DoubleValue Lambda {
|
---|
| 59 | get { return LambdaParameter.Value; }
|
---|
| 60 | set { LambdaParameter.Value = value; }
|
---|
[13930] | 61 | }
|
---|
[13927] | 62 | #endregion
|
---|
| 63 |
|
---|
| 64 | [StorableConstructor]
|
---|
[16565] | 65 | private ElasticNetLinearRegression(StorableConstructorFlag _) : base(_) { }
|
---|
[13927] | 66 | private ElasticNetLinearRegression(ElasticNetLinearRegression original, Cloner cloner)
|
---|
| 67 | : base(original, cloner) {
|
---|
| 68 | }
|
---|
[13940] | 69 | public ElasticNetLinearRegression()
|
---|
| 70 | : base() {
|
---|
[13927] | 71 | Problem = new RegressionProblem();
|
---|
[14377] | 72 | Parameters.Add(new FixedValueParameter<DoubleValue>(PenalityParameterName, "Penalty factor (alpha) for balancing between ridge (0.0) and lasso (1.0) regression", new DoubleValue(0.5)));
|
---|
[14395] | 73 | Parameters.Add(new OptionalValueParameter<DoubleValue>(LambdaParameterName, "Optional: the value of lambda for which to calculate an elastic-net solution. lambda == null => calculate the whole path of all lambdas"));
|
---|
[13927] | 74 | }
|
---|
| 75 |
|
---|
| 76 | [StorableHook(HookType.AfterDeserialization)]
|
---|
| 77 | private void AfterDeserialization() { }
|
---|
| 78 |
|
---|
| 79 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
| 80 | return new ElasticNetLinearRegression(this, cloner);
|
---|
| 81 | }
|
---|
| 82 |
|
---|
[14674] | 83 | protected override void Run(CancellationToken cancellationToken) {
|
---|
[14395] | 84 | if (Lambda == null) {
|
---|
[13930] | 85 | CreateSolutionPath();
|
---|
| 86 | } else {
|
---|
[14395] | 87 | CreateSolution(Lambda.Value);
|
---|
[13930] | 88 | }
|
---|
| 89 | }
|
---|
| 90 |
|
---|
[17991] | 91 | private void CreateSolution(double lambda) {
|
---|
[14225] | 92 | double trainNMSE;
|
---|
| 93 | double testNMSE;
|
---|
[15046] | 94 | var coeff = CalculateModelCoefficients(Problem.ProblemData, Penality, lambda, out trainNMSE, out testNMSE);
|
---|
[14225] | 95 | Results.Add(new Result("NMSE (train)", new DoubleValue(trainNMSE)));
|
---|
| 96 | Results.Add(new Result("NMSE (test)", new DoubleValue(testNMSE)));
|
---|
[13930] | 97 |
|
---|
[15046] | 98 | var solution = CreateSymbolicSolution(coeff, Problem.ProblemData);
|
---|
| 99 | Results.Add(new Result(solution.Name, solution.Description, solution));
|
---|
[17991] | 100 |
|
---|
| 101 | var xy = Problem.ProblemData.Dataset.ToArray(Problem.ProblemData.AllowedInputVariables.Concat(new[] { Problem.ProblemData.TargetVariable }), Problem.ProblemData.TrainingIndices);
|
---|
| 102 | // prepare xy for calculation of parameter statistics
|
---|
| 103 | // the last coefficient is the offset
|
---|
| 104 | var resid = new double[xy.GetLength(0)];
|
---|
| 105 | for (int r = 0; r < xy.GetLength(0); r++) {
|
---|
| 106 | resid[r] = xy[r, coeff.Length - 1] - coeff[coeff.Length - 1];
|
---|
| 107 | xy[r, coeff.Length - 1] = 1.0;
|
---|
| 108 | }
|
---|
| 109 | var statistics = Statistics.CalculateParameterStatistics(xy, coeff, resid);
|
---|
| 110 | Results.AddOrUpdateResult("Statistics", statistics.AsResultCollection(Problem.ProblemData.AllowedInputVariables.Concat(new string[] { "<const>" })));
|
---|
[15046] | 111 | }
|
---|
| 112 |
|
---|
| 113 | public static IRegressionSolution CreateSymbolicSolution(double[] coeff, IRegressionProblemData problemData) {
|
---|
| 114 | var ds = problemData.Dataset;
|
---|
| 115 | var allVariables = problemData.AllowedInputVariables.ToArray();
|
---|
[15023] | 116 | var doubleVariables = allVariables.Where(ds.VariableHasType<double>);
|
---|
| 117 | var factorVariableNames = allVariables.Where(ds.VariableHasType<string>);
|
---|
| 118 | var factorVariablesAndValues = ds.GetFactorVariableValues(factorVariableNames, Enumerable.Range(0, ds.Rows)); // must consider all factor values (in train and test set)
|
---|
[13930] | 119 |
|
---|
[15023] | 120 | List<KeyValuePair<string, IEnumerable<string>>> remainingFactorVariablesAndValues = new List<KeyValuePair<string, IEnumerable<string>>>();
|
---|
| 121 | List<double> factorCoeff = new List<double>();
|
---|
| 122 | List<string> remainingDoubleVariables = new List<string>();
|
---|
| 123 | List<double> doubleVarCoeff = new List<double>();
|
---|
[13930] | 124 |
|
---|
[15023] | 125 | {
|
---|
| 126 | int i = 0;
|
---|
| 127 | // find factor varibles & value combinations with non-zero coeff
|
---|
| 128 | foreach (var factorVarAndValues in factorVariablesAndValues) {
|
---|
| 129 | var l = new List<string>();
|
---|
| 130 | foreach (var factorValue in factorVarAndValues.Value) {
|
---|
| 131 | if (!coeff[i].IsAlmost(0.0)) {
|
---|
| 132 | l.Add(factorValue);
|
---|
| 133 | factorCoeff.Add(coeff[i]);
|
---|
| 134 | }
|
---|
| 135 | i++;
|
---|
| 136 | }
|
---|
| 137 | if (l.Any()) remainingFactorVariablesAndValues.Add(new KeyValuePair<string, IEnumerable<string>>(factorVarAndValues.Key, l));
|
---|
| 138 | }
|
---|
| 139 | // find double variables with non-zero coeff
|
---|
| 140 | foreach (var doubleVar in doubleVariables) {
|
---|
| 141 | if (!coeff[i].IsAlmost(0.0)) {
|
---|
| 142 | remainingDoubleVariables.Add(doubleVar);
|
---|
| 143 | doubleVarCoeff.Add(coeff[i]);
|
---|
| 144 | }
|
---|
| 145 | i++;
|
---|
| 146 | }
|
---|
| 147 | }
|
---|
| 148 | var tree = LinearModelToTreeConverter.CreateTree(
|
---|
[15046] | 149 | remainingFactorVariablesAndValues, factorCoeff.ToArray(),
|
---|
[15023] | 150 | remainingDoubleVariables.ToArray(), doubleVarCoeff.ToArray(),
|
---|
| 151 | coeff.Last());
|
---|
[13930] | 152 |
|
---|
[14395] | 153 |
|
---|
[13961] | 154 | SymbolicRegressionSolution solution = new SymbolicRegressionSolution(
|
---|
[15046] | 155 | new SymbolicRegressionModel(problemData.TargetVariable, tree, new SymbolicDataAnalysisExpressionTreeInterpreter()),
|
---|
| 156 | (IRegressionProblemData)problemData.Clone());
|
---|
[13930] | 157 | solution.Model.Name = "Elastic-net Linear Regression Model";
|
---|
| 158 | solution.Name = "Elastic-net Linear Regression Solution";
|
---|
| 159 |
|
---|
[15046] | 160 | return solution;
|
---|
[13930] | 161 | }
|
---|
| 162 |
|
---|
| 163 | private void CreateSolutionPath() {
|
---|
[13927] | 164 | double[] lambda;
|
---|
[14225] | 165 | double[] trainNMSE;
|
---|
| 166 | double[] testNMSE;
|
---|
[13927] | 167 | double[,] coeff;
|
---|
| 168 | double[] intercept;
|
---|
[14225] | 169 | RunElasticNetLinearRegression(Problem.ProblemData, Penality, out lambda, out trainNMSE, out testNMSE, out coeff, out intercept);
|
---|
[13927] | 170 |
|
---|
[14375] | 171 | var coeffTable = new IndexedDataTable<double>("Coefficients", "The paths of standarized coefficient values over different lambda values");
|
---|
[14374] | 172 | coeffTable.VisualProperties.YAxisMaximumAuto = false;
|
---|
| 173 | coeffTable.VisualProperties.YAxisMinimumAuto = false;
|
---|
| 174 | coeffTable.VisualProperties.XAxisMaximumAuto = false;
|
---|
| 175 | coeffTable.VisualProperties.XAxisMinimumAuto = false;
|
---|
| 176 |
|
---|
| 177 | coeffTable.VisualProperties.XAxisLogScale = true;
|
---|
[14395] | 178 | coeffTable.VisualProperties.XAxisTitle = "Lambda";
|
---|
[14375] | 179 | coeffTable.VisualProperties.YAxisTitle = "Coefficients";
|
---|
[14374] | 180 | coeffTable.VisualProperties.SecondYAxisTitle = "Number of variables";
|
---|
| 181 |
|
---|
[13927] | 182 | var nLambdas = lambda.Length;
|
---|
| 183 | var nCoeff = coeff.GetLength(1);
|
---|
[14374] | 184 | var dataRows = new IndexedDataRow<double>[nCoeff];
|
---|
[13928] | 185 | var allowedVars = Problem.ProblemData.AllowedInputVariables.ToArray();
|
---|
[14373] | 186 | var numNonZeroCoeffs = new int[nLambdas];
|
---|
[15023] | 187 |
|
---|
| 188 | var ds = Problem.ProblemData.Dataset;
|
---|
| 189 | var doubleVariables = allowedVars.Where(ds.VariableHasType<double>);
|
---|
| 190 | var factorVariableNames = allowedVars.Where(ds.VariableHasType<string>);
|
---|
| 191 | var factorVariablesAndValues = ds.GetFactorVariableValues(factorVariableNames, Enumerable.Range(0, ds.Rows)); // must consider all factor values (in train and test set)
|
---|
| 192 | {
|
---|
| 193 | int i = 0;
|
---|
| 194 | foreach (var factorVariableAndValues in factorVariablesAndValues) {
|
---|
| 195 | foreach (var factorValue in factorVariableAndValues.Value) {
|
---|
| 196 | double sigma = ds.GetStringValues(factorVariableAndValues.Key)
|
---|
| 197 | .Select(s => s == factorValue ? 1.0 : 0.0)
|
---|
| 198 | .StandardDeviation(); // calc std dev of binary indicator
|
---|
| 199 | var path = Enumerable.Range(0, nLambdas).Select(r => Tuple.Create(lambda[r], coeff[r, i] * sigma)).ToArray();
|
---|
| 200 | dataRows[i] = new IndexedDataRow<double>(factorVariableAndValues.Key + "=" + factorValue, factorVariableAndValues.Key + "=" + factorValue, path);
|
---|
| 201 | i++;
|
---|
| 202 | }
|
---|
| 203 | }
|
---|
| 204 |
|
---|
| 205 | foreach (var doubleVariable in doubleVariables) {
|
---|
| 206 | double sigma = ds.GetDoubleValues(doubleVariable).StandardDeviation();
|
---|
| 207 | var path = Enumerable.Range(0, nLambdas).Select(r => Tuple.Create(lambda[r], coeff[r, i] * sigma)).ToArray();
|
---|
| 208 | dataRows[i] = new IndexedDataRow<double>(doubleVariable, doubleVariable, path);
|
---|
| 209 | i++;
|
---|
| 210 | }
|
---|
| 211 | // add to coeffTable by total weight (larger area under the curve => more important);
|
---|
| 212 | foreach (var r in dataRows.OrderByDescending(r => r.Values.Select(t => t.Item2).Sum(x => Math.Abs(x)))) {
|
---|
| 213 | coeffTable.Rows.Add(r);
|
---|
| 214 | }
|
---|
[13927] | 215 | }
|
---|
[14844] | 216 |
|
---|
[14373] | 217 | for (int i = 0; i < coeff.GetLength(0); i++) {
|
---|
| 218 | for (int j = 0; j < coeff.GetLength(1); j++) {
|
---|
| 219 | if (!coeff[i, j].IsAlmost(0.0)) {
|
---|
| 220 | numNonZeroCoeffs[i]++;
|
---|
| 221 | }
|
---|
| 222 | }
|
---|
| 223 | }
|
---|
[14374] | 224 | if (lambda.Length > 2) {
|
---|
| 225 | coeffTable.VisualProperties.XAxisMinimumFixedValue = Math.Pow(10, Math.Floor(Math.Log10(lambda.Last())));
|
---|
| 226 | coeffTable.VisualProperties.XAxisMaximumFixedValue = Math.Pow(10, Math.Ceiling(Math.Log10(lambda.Skip(1).First())));
|
---|
| 227 | }
|
---|
| 228 | coeffTable.Rows.Add(new IndexedDataRow<double>("Number of variables", "The number of non-zero coefficients for each step in the path", lambda.Zip(numNonZeroCoeffs, (l, v) => Tuple.Create(l, (double)v))));
|
---|
| 229 | coeffTable.Rows["Number of variables"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
|
---|
| 230 | coeffTable.Rows["Number of variables"].VisualProperties.SecondYAxis = true;
|
---|
[13927] | 231 |
|
---|
| 232 | Results.Add(new Result(coeffTable.Name, coeffTable.Description, coeffTable));
|
---|
| 233 |
|
---|
[14375] | 234 | var errorTable = new IndexedDataTable<double>("NMSE", "Path of NMSE values over different lambda values");
|
---|
| 235 | errorTable.VisualProperties.YAxisMaximumAuto = false;
|
---|
| 236 | errorTable.VisualProperties.YAxisMinimumAuto = false;
|
---|
| 237 | errorTable.VisualProperties.XAxisMaximumAuto = false;
|
---|
| 238 | errorTable.VisualProperties.XAxisMinimumAuto = false;
|
---|
[13940] | 239 |
|
---|
[14375] | 240 | errorTable.VisualProperties.YAxisMinimumFixedValue = 0;
|
---|
| 241 | errorTable.VisualProperties.YAxisMaximumFixedValue = 1.0;
|
---|
| 242 | errorTable.VisualProperties.XAxisLogScale = true;
|
---|
[14395] | 243 | errorTable.VisualProperties.XAxisTitle = "Lambda";
|
---|
[14375] | 244 | errorTable.VisualProperties.YAxisTitle = "Normalized mean of squared errors (NMSE)";
|
---|
[14395] | 245 | errorTable.VisualProperties.SecondYAxisTitle = "Number of variables";
|
---|
[14375] | 246 | errorTable.Rows.Add(new IndexedDataRow<double>("NMSE (train)", "Path of NMSE values over different lambda values", lambda.Zip(trainNMSE, (l, v) => Tuple.Create(l, v))));
|
---|
| 247 | errorTable.Rows.Add(new IndexedDataRow<double>("NMSE (test)", "Path of NMSE values over different lambda values", lambda.Zip(testNMSE, (l, v) => Tuple.Create(l, v))));
|
---|
| 248 | errorTable.Rows.Add(new IndexedDataRow<double>("Number of variables", "The number of non-zero coefficients for each step in the path", lambda.Zip(numNonZeroCoeffs, (l, v) => Tuple.Create(l, (double)v))));
|
---|
[13940] | 249 | if (lambda.Length > 2) {
|
---|
[14375] | 250 | errorTable.VisualProperties.XAxisMinimumFixedValue = Math.Pow(10, Math.Floor(Math.Log10(lambda.Last())));
|
---|
| 251 | errorTable.VisualProperties.XAxisMaximumFixedValue = Math.Pow(10, Math.Ceiling(Math.Log10(lambda.Skip(1).First())));
|
---|
[13940] | 252 | }
|
---|
[14375] | 253 | errorTable.Rows["NMSE (train)"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
|
---|
| 254 | errorTable.Rows["NMSE (test)"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
|
---|
| 255 | errorTable.Rows["Number of variables"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
|
---|
| 256 | errorTable.Rows["Number of variables"].VisualProperties.SecondYAxis = true;
|
---|
[14395] | 257 |
|
---|
[14375] | 258 | Results.Add(new Result(errorTable.Name, errorTable.Description, errorTable));
|
---|
[13927] | 259 | }
|
---|
| 260 |
|
---|
[15046] | 261 | public static double[] CalculateModelCoefficients(IRegressionProblemData problemData, double penalty, double lambda,
|
---|
[14225] | 262 | out double trainNMSE, out double testNMSE,
|
---|
[13927] | 263 | double coeffLowerBound = double.NegativeInfinity, double coeffUpperBound = double.PositiveInfinity) {
|
---|
[14225] | 264 | double[] trainNMSEs;
|
---|
| 265 | double[] testNMSEs;
|
---|
[13927] | 266 | // run for exactly one lambda
|
---|
[15046] | 267 | var coeffs = CalculateModelCoefficients(problemData, penalty, new double[] { lambda }, out trainNMSEs, out testNMSEs, coeffLowerBound, coeffUpperBound);
|
---|
[14225] | 268 | trainNMSE = trainNMSEs[0];
|
---|
| 269 | testNMSE = testNMSEs[0];
|
---|
[13927] | 270 | return coeffs[0];
|
---|
| 271 | }
|
---|
[15046] | 272 | public static double[][] CalculateModelCoefficients(IRegressionProblemData problemData, double penalty, double[] lambda,
|
---|
[14370] | 273 | out double[] trainNMSEs, out double[] testNMSEs,
|
---|
[13927] | 274 | double coeffLowerBound = double.NegativeInfinity, double coeffUpperBound = double.PositiveInfinity,
|
---|
| 275 | int maxVars = -1) {
|
---|
| 276 | // run for multiple user-supplied lambdas
|
---|
| 277 | double[,] coeff;
|
---|
| 278 | double[] intercept;
|
---|
[14370] | 279 | RunElasticNetLinearRegression(problemData, penalty, lambda.Length, 1.0, lambda, out lambda, out trainNMSEs, out testNMSEs, out coeff, out intercept, coeffLowerBound, coeffUpperBound, maxVars);
|
---|
[13927] | 280 |
|
---|
| 281 | int nRows = intercept.Length;
|
---|
| 282 | int nCols = coeff.GetLength(1) + 1;
|
---|
| 283 | double[][] sols = new double[nRows][];
|
---|
| 284 | for (int solIdx = 0; solIdx < nRows; solIdx++) {
|
---|
| 285 | sols[solIdx] = new double[nCols];
|
---|
| 286 | for (int cIdx = 0; cIdx < nCols - 1; cIdx++) {
|
---|
| 287 | sols[solIdx][cIdx] = coeff[solIdx, cIdx];
|
---|
| 288 | }
|
---|
| 289 | sols[solIdx][nCols - 1] = intercept[solIdx];
|
---|
| 290 | }
|
---|
| 291 | return sols;
|
---|
| 292 | }
|
---|
| 293 |
|
---|
| 294 | public static void RunElasticNetLinearRegression(IRegressionProblemData problemData, double penalty,
|
---|
[14225] | 295 | out double[] lambda, out double[] trainNMSE, out double[] testNMSE, out double[,] coeff, out double[] intercept,
|
---|
[13927] | 296 | double coeffLowerBound = double.NegativeInfinity, double coeffUpperBound = double.PositiveInfinity,
|
---|
| 297 | int maxVars = -1
|
---|
| 298 | ) {
|
---|
| 299 | double[] userLambda = new double[0];
|
---|
| 300 | // automatically determine lambda values (maximum 100 different lambda values)
|
---|
[14225] | 301 | RunElasticNetLinearRegression(problemData, penalty, 100, 0.0, userLambda, out lambda, out trainNMSE, out testNMSE, out coeff, out intercept, coeffLowerBound, coeffUpperBound, maxVars);
|
---|
[13927] | 302 | }
|
---|
| 303 |
|
---|
| 304 | /// <summary>
|
---|
| 305 | /// Elastic net with squared-error-loss for dense predictor matrix, runs the full path of all lambdas
|
---|
| 306 | /// </summary>
|
---|
| 307 | /// <param name="problemData">Predictor target matrix x and target vector y</param>
|
---|
| 308 | /// <param name="penalty">Penalty for balance between ridge (0.0) and lasso (1.0) regression</param>
|
---|
| 309 | /// <param name="nlam">Maximum number of lambda values (default 100)</param>
|
---|
| 310 | /// <param name="flmin">User control of lambda values (<1.0 => minimum lambda = flmin * (largest lambda value), >= 1.0 => use supplied lambda values</param>
|
---|
| 311 | /// <param name="ulam">User supplied lambda values</param>
|
---|
| 312 | /// <param name="lambda">Output lambda values</param>
|
---|
[14225] | 313 | /// <param name="trainNMSE">Vector of normalized mean of squared error (NMSE = Variance(res) / Variance(y)) values on the training set for each set of coefficients along the path</param>
|
---|
| 314 | /// <param name="testNMSE">Vector of normalized mean of squared error (NMSE = Variance(res) / Variance(y)) values on the test set for each set of coefficients along the path</param>
|
---|
[13927] | 315 | /// <param name="coeff">Vector of coefficient vectors for each solution along the path</param>
|
---|
| 316 | /// <param name="intercept">Vector of intercepts for each solution along the path</param>
|
---|
| 317 | /// <param name="coeffLowerBound">Optional lower bound for all coefficients</param>
|
---|
| 318 | /// <param name="coeffUpperBound">Optional upper bound for all coefficients</param>
|
---|
| 319 | /// <param name="maxVars">Maximum allowed number of variables in each solution along the path (-1 => all variables are allowed)</param>
|
---|
| 320 | private static void RunElasticNetLinearRegression(IRegressionProblemData problemData, double penalty,
|
---|
[14225] | 321 | int nlam, double flmin, double[] ulam, out double[] lambda, out double[] trainNMSE, out double[] testNMSE, out double[,] coeff, out double[] intercept,
|
---|
[13927] | 322 | double coeffLowerBound = double.NegativeInfinity, double coeffUpperBound = double.PositiveInfinity,
|
---|
| 323 | int maxVars = -1
|
---|
| 324 | ) {
|
---|
| 325 | if (penalty < 0.0 || penalty > 1.0) throw new ArgumentException("0 <= penalty <= 1", "penalty");
|
---|
| 326 |
|
---|
[13929] | 327 | double[,] trainX;
|
---|
| 328 | double[,] testX;
|
---|
| 329 | double[] trainY;
|
---|
| 330 | double[] testY;
|
---|
[13927] | 331 |
|
---|
[14395] | 332 | PrepareData(problemData, out trainX, out trainY, out testX, out testY);
|
---|
| 333 | var numTrainObs = trainX.GetLength(1);
|
---|
| 334 | var numTestObs = testX.GetLength(1);
|
---|
| 335 | var numVars = trainX.GetLength(0);
|
---|
| 336 |
|
---|
[13927] | 337 | int ka = 1; // => covariance updating algorithm
|
---|
| 338 | double parm = penalty;
|
---|
[13929] | 339 | double[] w = Enumerable.Repeat(1.0, numTrainObs).ToArray(); // all observations have the same weight
|
---|
[13927] | 340 | int[] jd = new int[1]; // do not force to use any of the variables
|
---|
| 341 | double[] vp = Enumerable.Repeat(1.0, numVars).ToArray(); // all predictor variables are unpenalized
|
---|
| 342 | double[,] cl = new double[numVars, 2]; // use the same bounds for all coefficients
|
---|
| 343 | for (int i = 0; i < numVars; i++) {
|
---|
| 344 | cl[i, 0] = coeffLowerBound;
|
---|
| 345 | cl[i, 1] = coeffUpperBound;
|
---|
| 346 | }
|
---|
| 347 |
|
---|
| 348 | int ne = maxVars > 0 ? maxVars : numVars;
|
---|
| 349 | int nx = numVars;
|
---|
| 350 | double thr = 1.0e-5; // default value as recommended in glmnet
|
---|
[13940] | 351 | int isd = 1; // => regression on standardized predictor variables
|
---|
[13927] | 352 | int intr = 1; // => do include intercept in model
|
---|
| 353 | int maxit = 100000; // default value as recommended in glmnet
|
---|
| 354 | // outputs
|
---|
| 355 | int lmu = -1;
|
---|
| 356 | double[,] ca;
|
---|
| 357 | int[] ia;
|
---|
| 358 | int[] nin;
|
---|
| 359 | int nlp = -99;
|
---|
| 360 | int jerr = -99;
|
---|
[14225] | 361 | double[] trainR2;
|
---|
[14370] | 362 | Glmnet.elnet(ka, parm, numTrainObs, numVars, trainX, trainY, w, jd, vp, cl, ne, nx, nlam, flmin, ulam, thr, isd, intr, maxit, out lmu, out intercept, out ca, out ia, out nin, out trainR2, out lambda, out nlp, out jerr);
|
---|
[13927] | 363 |
|
---|
[14225] | 364 | trainNMSE = new double[lmu]; // elnet returns R**2 as 1 - NMSE
|
---|
| 365 | testNMSE = new double[lmu];
|
---|
[13927] | 366 | coeff = new double[lmu, numVars];
|
---|
| 367 | for (int solIdx = 0; solIdx < lmu; solIdx++) {
|
---|
[14225] | 368 | trainNMSE[solIdx] = 1.0 - trainR2[solIdx];
|
---|
| 369 |
|
---|
[13927] | 370 | // uncompress coefficients of solution
|
---|
| 371 | int selectedNin = nin[solIdx];
|
---|
| 372 | double[] coefficients;
|
---|
| 373 | double[] selectedCa = new double[nx];
|
---|
[13929] | 374 | for (int i = 0; i < nx; i++) {
|
---|
| 375 | selectedCa[i] = ca[solIdx, i];
|
---|
| 376 | }
|
---|
[13927] | 377 |
|
---|
[14225] | 378 | // apply to test set to calculate test NMSE values for each lambda step
|
---|
[13929] | 379 | double[] fn;
|
---|
[14370] | 380 | Glmnet.modval(intercept[solIdx], selectedCa, ia, selectedNin, numTestObs, testX, out fn);
|
---|
[13929] | 381 | OnlineCalculatorError error;
|
---|
[14225] | 382 | var nmse = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(testY, fn, out error);
|
---|
[14461] | 383 | if (error != OnlineCalculatorError.None) nmse = double.NaN;
|
---|
[14225] | 384 | testNMSE[solIdx] = nmse;
|
---|
[13929] | 385 |
|
---|
| 386 | // uncompress coefficients
|
---|
[14370] | 387 | Glmnet.uncomp(numVars, selectedCa, ia, selectedNin, out coefficients);
|
---|
[13927] | 388 | for (int i = 0; i < coefficients.Length; i++) {
|
---|
| 389 | coeff[solIdx, i] = coefficients[i];
|
---|
| 390 | }
|
---|
| 391 | }
|
---|
| 392 | }
|
---|
| 393 |
|
---|
[14395] | 394 | private static void PrepareData(IRegressionProblemData problemData, out double[,] trainX, out double[] trainY,
|
---|
| 395 | out double[,] testX, out double[] testY) {
|
---|
[15023] | 396 | var ds = problemData.Dataset;
|
---|
| 397 | var targetVariable = problemData.TargetVariable;
|
---|
| 398 | var allowedInputs = problemData.AllowedInputVariables;
|
---|
| 399 | trainX = PrepareInputData(ds, allowedInputs, problemData.TrainingIndices);
|
---|
| 400 | trainY = ds.GetDoubleValues(targetVariable, problemData.TrainingIndices).ToArray();
|
---|
[13927] | 401 |
|
---|
[15023] | 402 | testX = PrepareInputData(ds, allowedInputs, problemData.TestIndices);
|
---|
| 403 | testY = ds.GetDoubleValues(targetVariable, problemData.TestIndices).ToArray();
|
---|
[13927] | 404 | }
|
---|
[15023] | 405 |
|
---|
| 406 | private static double[,] PrepareInputData(IDataset ds, IEnumerable<string> allowedInputs, IEnumerable<int> rows) {
|
---|
| 407 | var doubleVariables = allowedInputs.Where(ds.VariableHasType<double>);
|
---|
| 408 | var factorVariableNames = allowedInputs.Where(ds.VariableHasType<string>);
|
---|
| 409 | var factorVariables = ds.GetFactorVariableValues(factorVariableNames, Enumerable.Range(0, ds.Rows)); // must consider all factor values (in train and test set)
|
---|
| 410 | double[,] binaryMatrix = ds.ToArray(factorVariables, rows);
|
---|
| 411 | double[,] doubleVarMatrix = ds.ToArray(doubleVariables, rows);
|
---|
| 412 | var x = binaryMatrix.HorzCat(doubleVarMatrix);
|
---|
| 413 | return x.Transpose();
|
---|
| 414 | }
|
---|
[13927] | 415 | }
|
---|
| 416 | }
|
---|