Changeset 15967 for branches/M5Regression

Timestamp:

06/20/18 09:53:28 (7 years ago)

Author:

bwerth

Message:

#2847 added logistic dampening and some minor changes

Location:

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4

Files:

• HeuristicLab.Algorithms.DataAnalysis-3.4.csproj (modified) (2 diffs)
• M5Regression/LeafModels/ComponentReducedLinearModel.cs (modified) (3 diffs)
• M5Regression/LeafModels/DampenedModel.cs (added)
• M5Regression/LeafModels/PreconstructedLinearModel.cs (modified) (6 diffs)
• M5Regression/LeafTypes/ComponentReductionLinearLeaf.cs (modified) (2 diffs)
• M5Regression/LeafTypes/LeafBase.cs (modified) (4 diffs)
• M5Regression/LeafTypes/LinearLeaf.cs (modified) (2 diffs)
• M5Regression/LeafTypes/M5Leaf.cs (modified) (2 diffs)
• M5Regression/M5Regression.cs (modified) (1 diff)

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj

@@ -372 +372 @@
     <Compile Include="M5Regression\Interfaces\ISplitter.cs" />
     <Compile Include="M5Regression\LeafModels\ComponentReducedLinearModel.cs" />
-    <Compile Include="M5Regression\LeafModels\DampenedLinearModel.cs" />
+    <Compile Include="M5Regression\LeafModels\DampenedModel.cs" />
     <Compile Include="M5Regression\LeafModels\PreconstructedLinearModel.cs" />
     <Compile Include="M5Regression\LeafTypes\ComplexLeaf.cs" />
@@ -380 +380 @@
     <Compile Include="M5Regression\LeafTypes\GaussianProcessLeaf.cs" />
     <Compile Include="M5Regression\LeafTypes\LinearLeaf.cs" />
-    <Compile Include="M5Regression\LeafTypes\LogisticLeaf.cs" />
     <Compile Include="M5Regression\LeafTypes\M5Leaf.cs" />
     <Compile Include="M5Regression\LeafTypes\M5regLeaf.cs" />

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/LeafModels/ComponentReducedLinearModel.cs

@@ -28 +28 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   [StorableClass]
-  public class ComponentReducedLinearModel : RegressionModel, IConfidenceRegressionModel {
+  public class ComponentReducedLinearModel : RegressionModel {
     [Storable]
-    private IConfidenceRegressionModel Model;
+    private IRegressionModel Model;
     [Storable]
     private PrincipleComponentTransformation Pca;
@@ -40 +40 @@
       Pca = cloner.Clone(original.Pca);
     }
-    public ComponentReducedLinearModel(string targetVariable, IConfidenceRegressionModel model, PrincipleComponentTransformation pca) : base(targetVariable) {
+    public ComponentReducedLinearModel(string targetVariable, IRegressionModel model, PrincipleComponentTransformation pca) : base(targetVariable) {
       Model = model;
       Pca = pca;
@@ -56 +56 @@
     }
     public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      return new ConfidenceRegressionSolution(this, problemData);
-    }
-    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
-      var data = ReduceDataset(dataset, rows.ToArray());
-      return Model.GetEstimatedVariances(Pca.TransformDataset(data), Enumerable.Range(0, data.Rows));
+      return new RegressionSolution(this, problemData);
     }
 

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/LeafModels/PreconstructedLinearModel.cs

@@ -31 +31 @@
   //mulitdimensional extension of http://www2.stat.duke.edu/~tjl13/s101/slides/unit6lec3H.pdf
   [StorableClass]
-  internal sealed class PreconstructedLinearModel : RegressionModel, IConfidenceRegressionModel {
+  internal sealed class PreconstructedLinearModel : RegressionModel {
     [Storable]
     public Dictionary<string, double> Coefficients { get; private set; }
     [Storable]
     public double Intercept { get; private set; }
-    [Storable]
-    private Dictionary<string, double> Means { get; set; }
-    [Storable]
-    private Dictionary<string, double> Variances { get; set; }
-    [Storable]
-    private double ResidualVariance { get; set; }
-    [Storable]
-    private int SampleSize { get; set; }
 
     public override IEnumerable<string> VariablesUsedForPrediction {
@@ -54 +46 @@
       if (original.Coefficients != null) Coefficients = original.Coefficients.ToDictionary(x => x.Key, x => x.Value);
       Intercept = original.Intercept;
-      if (original.Means != null) Means = original.Means.ToDictionary(x => x.Key, x => x.Value);
-      if (original.Variances != null) Variances = original.Variances.ToDictionary(x => x.Key, x => x.Value);
-      ResidualVariance = original.ResidualVariance;
-      SampleSize = original.SampleSize;
     }
-    public PreconstructedLinearModel(Dictionary<string, double> means, Dictionary<string, double> variances, Dictionary<string, double> coefficients, double intercept, string targetvariable, double residualVariance = 0, double sampleSize = 0) : base(targetvariable) {
+    public PreconstructedLinearModel(Dictionary<string, double> coefficients, double intercept, string targetvariable) : base(targetvariable) {
       Coefficients = coefficients;
       Intercept = intercept;
-      Variances = variances;
-      Means = means;
-      ResidualVariance = 0;
-      SampleSize = 0;
     }
     public PreconstructedLinearModel(double intercept, string targetvariable) : base(targetvariable) {
       Coefficients = new Dictionary<string, double>();
       Intercept = intercept;
-      Variances = new Dictionary<string, double>();
-      ResidualVariance = 0;
-      SampleSize = 0;
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -79 +60 @@
     #endregion
 
-    public static PreconstructedLinearModel CreateConfidenceLinearModel(IRegressionProblemData pd, out double rmse, out double cvRmse) {
-      rmse = double.NaN;
-      cvRmse = double.NaN;
-      return AlternativeCalculation(pd);
+    public static PreconstructedLinearModel CreateLinearModel(IRegressionProblemData pd, out double rmse) {
+      return AlternativeCalculation(pd, out rmse);
     }
 
-    private static PreconstructedLinearModel ClassicCalculation(IRegressionProblemData pd, out double rmse, out double cvRmse) {
+    private static PreconstructedLinearModel ClassicCalculation(IRegressionProblemData pd) {
       var inputMatrix = pd.Dataset.ToArray(pd.AllowedInputVariables.Concat(new[] {
         pd.TargetVariable
@@ -98 +77 @@
       alglib.lrbuild(inputMatrix, inputMatrix.GetLength(0), nFeatures, out retVal, out lm, out ar);
       if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
-      rmse = ar.rmserror;
-      cvRmse = ar.cvrmserror;
 
       alglib.lrunpack(lm, out coefficients, out nFeatures);
-
-
-      var means = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n).Average());
-      var variances = pd.AllowedInputVariables.ToDictionary(n => n, n => pd.Dataset.GetDoubleValues(n).Variance());
       var coeffs = pd.AllowedInputVariables.Zip(coefficients, (s, d) => new {s, d}).ToDictionary(x => x.s, x => x.d);
-      var res = new PreconstructedLinearModel(means, variances, coeffs, coefficients[nFeatures], pd.TargetVariable);
-
-      res.ResidualVariance = pd.TargetVariableValues.Zip(res.GetEstimatedValues(pd.Dataset, pd.TrainingIndices), (x, y) => x - y).Variance();
-      res.SampleSize = pd.TrainingIndices.Count();
+      var res = new PreconstructedLinearModel(coeffs, coefficients[nFeatures], pd.TargetVariable);
       return res;
     }
 
-    private static PreconstructedLinearModel AlternativeCalculation(IRegressionProblemData pd) {
-      var means = pd.AllowedInputVariables.ToDictionary(n1 => n1, n1 => pd.Dataset.GetDoubleValues(n1).Average());
-      var variances = pd.AllowedInputVariables.ToDictionary(n1 => n1, n1 => pd.Dataset.GetDoubleValues(n1).Variance());
-      var cmean = pd.TargetVariableTrainingValues.Average();
+    private static PreconstructedLinearModel AlternativeCalculation(IRegressionProblemData pd, out double rmse) {
       var variables = pd.AllowedInputVariables.ToList();
       var n = variables.Count;
       var m = pd.TrainingIndices.Count();
 
-      //Set up X^T and y
+      //Set up X^T
       var inTr = new double[n + 1, m];
       for (var i = 0; i < n; i++) {
-        var v = variables[i];
-        var vdata = pd.Dataset.GetDoubleValues(v, pd.TrainingIndices).ToArray();
+        var vdata = pd.Dataset.GetDoubleValues(variables[i], pd.TrainingIndices).ToArray();
         for (var j = 0; j < m; j++) inTr[i, j] = vdata[j];
       }
-
       for (var i = 0; i < m; i++) inTr[n, i] = 1;
 
+      //Set up y
       var y = new double[m, 1];
       var ydata = pd.TargetVariableTrainingValues.ToArray();
@@ -138 +104 @@
       //Perform linear regression
       var aTy = new double[n + 1, 1];
-      alglib.rmatrixgemm(n + 1, 1, m, 1, inTr, 0, 0, 0, y, 0, 0, 0, 0, ref aTy, 0, 0); //aTy = inTr * y;
       var aTa = new double[n + 1, n + 1];
-      alglib.rmatrixgemm(n + 1, n + 1, m, 1, inTr, 0, 0, 0, inTr, 0, 0, 1, 0, ref aTa, 0, 0); //aTa = inTr * t(inTr) +aTa //
-      alglib.spdmatrixcholesky(ref aTa, n + 1, true);
+      var aTyVector = new double[n + 1];
       int info;
       alglib.densesolverreport report;
       double[] coefficients;
-      var aTyVector = new double[n + 1];
+
+      //Perform linear regression
+      alglib.rmatrixgemm(n + 1, 1, m, 1, inTr, 0, 0, 0, y, 0, 0, 0, 0, ref aTy, 0, 0); //aTy = inTr * y;
+      alglib.rmatrixgemm(n + 1, n + 1, m, 1, inTr, 0, 0, 0, inTr, 0, 0, 1, 0, ref aTa, 0, 0); //aTa = inTr * t(inTr) +aTa //
+      alglib.spdmatrixcholesky(ref aTa, n + 1, true);
       for (var i = 0; i < n + 1; i++) aTyVector[i] = aTy[i, 0];
       alglib.spdmatrixcholeskysolve(aTa, n + 1, true, aTyVector, out info, out report, out coefficients);
-      double rmse, cvrmse;
-      if (info != 1) return ClassicCalculation(pd, out rmse, out cvrmse);
 
-      //extract coefficients
-      var intercept = coefficients[n];
-      var coeffs = new Dictionary<string, double>();
-      for (var i = 0; i < n; i++) coeffs.Add(variables[i], coefficients[i]);
+      //if cholesky calculation fails fall bakc to classic linear regresseion 
+      if (info != 1) {
+        alglib.linearmodel lm;
+        alglib.lrreport ar;
+        int retVal;
+        var inputMatrix = pd.Dataset.ToArray(pd.AllowedInputVariables.Concat(new[] {
+          pd.TargetVariable
+        }), pd.AllIndices);
+        alglib.lrbuild(inputMatrix, inputMatrix.GetLength(0), n, out retVal, out lm, out ar);
+        if (retVal != 1) throw new ArgumentException("Error in calculation of linear regression solution");
+        alglib.lrunpack(lm, out coefficients, out n);
+      }
 
-      return new PreconstructedLinearModel(means, variances, coeffs, intercept, pd.TargetVariable);
+      var coeffs = Enumerable.Range(0, n).ToDictionary(i => variables[i], i => coefficients[i]);
+      var model = new PreconstructedLinearModel(coeffs, coefficients[n], pd.TargetVariable);
+      rmse = pd.TrainingIndices.Select(i => pd.Dataset.GetDoubleValue(pd.TargetVariable, i) - model.GetEstimatedValue(pd.Dataset, i)).Sum(r => r * r) / m;
+      rmse = Math.Sqrt(rmse);
+      return model;
     }
 
@@ -167 +145 @@
     }
 
-    public IEnumerable<double> GetEstimatedVariances(IDataset dataset, IEnumerable<int> rows) {
-      return rows.Select(i => GetEstimatedVariance(dataset, i));
-    }
-
     #region helpers
     private double GetEstimatedValue(IDataset dataset, int row) {
       return Intercept + (Coefficients.Count == 0 ? 0 : Coefficients.Sum(s => s.Value * dataset.GetDoubleValue(s.Key, row)));
     }
-    private double GetEstimatedVariance(IDataset dataset, int row) {
-      if (SampleSize == 0) return 0.0;
-      var sum = (from var in Variances let d = dataset.GetDoubleValue(var.Key, row) - Means[var.Key] select d * d / var.Value).Sum();
-      var res = ResidualVariance * (SampleSize - 1) / (SampleSize - 2) * (1.0 / SampleSize + sum / (SampleSize - 1));
-      if (double.IsInfinity(res) || double.IsNaN(res)) return 0.0;
-      return Math.Sqrt(res);
-    }
     #endregion
   }

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/LeafTypes/ComponentReductionLinearLeaf.cs

@@ -57 +57 @@
     #region IModelType
     public override bool ProvidesConfidence {
-      get { return true; }
+      get { return false; }
     }
     public override IRegressionModel Build(IRegressionProblemData pd, IRandom random,
@@ -71 +71 @@
         foreach (var v in pd2.InputVariables.CheckedItems.ToArray())
           pd2.InputVariables.SetItemCheckedState(v.Value, inputs.Contains(v.Value.Value));
-        double cvRmse;
         double rmse;
-        var model = PreconstructedLinearModel.CreateConfidenceLinearModel(pd2, out rmse, out cvRmse);
-        if (cvRmse > bestCvrmse) continue;
+        var model = PreconstructedLinearModel.CreateLinearModel(pd2, out rmse);
+        if (rmse > bestCvrmse) continue;
         bestModel = new ComponentReducedLinearModel(pd2.TargetVariable, model, pca);
         noParameters = i + 1;
-        bestCvrmse = cvRmse;
+        bestCvrmse = rmse;
       }
       return bestModel;

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/LeafTypes/LeafBase.cs

@@ -26 +26 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Parameters;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.Problems.DataAnalysis;
@@ -33 +35 @@
   public abstract class LeafBase : ParameterizedNamedItem, ILeafModel {
     public const string LeafBuildingStateVariableName = "LeafBuildingState";
+    public const string UseDampeningParameterName = "UseDampening";
+    private const string DampeningParameterName = "DampeningStrenght";
+
+    public IFixedValueParameter<DoubleValue> DampeningParameter {
+      get { return Parameters[DampeningParameterName] as IFixedValueParameter<DoubleValue>; }
+    }
+    public IFixedValueParameter<BoolValue> UseDampeningParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[UseDampeningParameterName]; }
+    }
+
+    public bool UseDampening {
+      get { return UseDampeningParameter.Value.Value; }
+    }
+    public double Dampening {
+      get { return DampeningParameter.Value.Value; }
+    }
 
     #region Constructors & Cloning
@@ -38 +56 @@
     protected LeafBase(bool deserializing) : base(deserializing) { }
     protected LeafBase(LeafBase original, Cloner cloner) : base(original, cloner) { }
-    public LeafBase() { }
+    protected LeafBase() {
+      Parameters.Add(new FixedValueParameter<BoolValue>(UseDampeningParameterName, "Whether logistic dampening should be used to prevent extreme extrapolation", new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(DampeningParameterName, "Determines the strenght of the logistic dampening. Must be > 0.0. Larger numbers make more conservative predictions.", new DoubleValue(1.5)));
+    }
     #endregion
 
@@ -73 +94 @@
       int numP;
       var model = Build(pd, parameters.Random, cancellation, out numP);
+      if (UseDampening && Dampening > 0.0) {
+        model = DampenedModel.DampenModel(model, pd, Dampening);
+      }
+
       numParams = numP;
       cancellation.ThrowIfCancellationRequested();

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/LeafTypes/LinearLeaf.cs

@@ -44 +44 @@
     #region IModelType
     public override bool ProvidesConfidence {
-      get { return true; }
+      get { return false; }
     }
     public override IRegressionModel Build(IRegressionProblemData pd, IRandom random, CancellationToken cancellationToken, out int noParameters) {
@@ -50 +50 @@
       double rmse, cvRmse;
       noParameters = pd.AllowedInputVariables.Count() + 1;
-      var res = PreconstructedLinearModel.CreateConfidenceLinearModel(pd, out rmse, out cvRmse);
-      return res;
+      var res = LinearRegression.CreateLinearRegressionSolution(pd, out rmse, out cvRmse);
+      return res.Model;
     }
 

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/LeafTypes/M5Leaf.cs

@@ -112 +112 @@
       pd2.TrainingPartition.Start = pd.TrainingPartition.Start;
 
-      return new PreconstructedLinearModel(means, variances, coeffs, intercept, pd.TargetVariable);
+      return new PreconstructedLinearModel(coeffs, intercept, pd.TargetVariable);
     }
 
@@ -124 +124 @@
       //
       //  double x1, x2;
-      //  var lm = PreconstructedLinearModel.CreateConfidenceLinearModel(pd2, out x1, out x2);
+      //  var lm = PreconstructedLinearModel.CreateLinearModel(pd2, out x1, out x2);
       //  intercept = lm.Intercept;
       //  return lm.Coefficients;

branches/M5Regression/HeuristicLab.Algorithms.DataAnalysis/3.4/M5Regression/M5Regression.cs

@@ -220 +220 @@
       var pruningRows = (IntArray)stateScope.Variables[PruningSetVariableName].Value;
       if (1 > trainingRows.Length)
-        return new PreconstructedLinearModel(new Dictionary<string, double>(), new Dictionary<string, double>(), new Dictionary<string, double>(), 0, regressionTreeParams.TargetVariable);
+        return new PreconstructedLinearModel(new Dictionary<string, double>(), 0, regressionTreeParams.TargetVariable);
       if (regressionTreeParams.MinLeafSize > trainingRows.Length) {
         var targets = regressionTreeParams.Data.GetDoubleValues(regressionTreeParams.TargetVariable).ToArray();
-        return new PreconstructedLinearModel(new Dictionary<string, double>(), new Dictionary<string, double>(), new Dictionary<string, double>(), targets.Average(), regressionTreeParams.TargetVariable, targets.Variance(), targets.Length);
+        return new PreconstructedLinearModel(new Dictionary<string, double>(), targets.Average(), regressionTreeParams.TargetVariable);
       }
       model.Build(trainingRows.ToArray(), pruningRows.ToArray(), stateScope, results, cancellationToken);