Changeset 12607 for branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs

Timestamp:

07/06/15 15:20:28 (9 years ago)

Author:

gkronber

Message:

#2261: also use line search function for the initial estimation f0, changed logistic regression loss function to match description in GBM paper, comments and code improvements

File:

• branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs (modified) (4 diffs)

branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs

@@ -28 +28 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
+  // Greedy Function Approximation: A Gradient Boosting Machine (page 9) 
   public class LogisticRegressionLoss : ILossFunction {
     public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred, IEnumerable<double> weight) {
@@ -36 +37 @@
       double s = 0;
       while (targetEnum.MoveNext() & predEnum.MoveNext() & weightEnum.MoveNext()) {
-        // assert target == 0 or target == 1
-        if (!targetEnum.Current.IsAlmost(0.0) && !targetEnum.Current.IsAlmost(1.0))
-          throw new NotSupportedException("labels must be 0 or 1 for logistic regression loss");
-        double f = Math.Max(-7, Math.Min(7, predEnum.Current)); // threshold for exponent
-        var probPos = Math.Exp(2 * f) / (1 + Math.Exp(2 * f));
-        s += weightEnum.Current * (-targetEnum.Current * Math.Log(probPos) - (1 - targetEnum.Current) * Math.Log(1 - probPos));
+        Debug.Assert(targetEnum.Current.IsAlmost(0.0) || targetEnum.Current.IsAlmost(1.0), "labels must be 0 or 1 for logistic regression loss");
+
+        var y = targetEnum.Current * 2 - 1; // y in {-1,1}
+        s += weightEnum.Current * Math.Log(1 + Math.Exp(-2 * y * predEnum.Current));
       }
       if (targetEnum.MoveNext() | predEnum.MoveNext() | weightEnum.MoveNext())
-        throw new ArgumentException("target, pred and weight have differing lengths");
+        throw new ArgumentException("target, pred and weight have different lengths");
 
       return s;
@@ -55 +54 @@
 
       while (targetEnum.MoveNext() & predEnum.MoveNext() & weightEnum.MoveNext()) {
-        // assert target == 0 or target == 1
-        if (!targetEnum.Current.IsAlmost(0.0) && !targetEnum.Current.IsAlmost(1.0))
-          throw new NotSupportedException("labels must be 0 or 1 for logistic regression loss");
-        double f = Math.Max(-7, Math.Min(7, predEnum.Current)); // threshold for exponent
-        var probPos = Math.Exp(2 * f) / (1 + Math.Exp(2 * f));
-        yield return weightEnum.Current * (targetEnum.Current - probPos) / (probPos * probPos - probPos);
+        Debug.Assert(targetEnum.Current.IsAlmost(0.0) || targetEnum.Current.IsAlmost(1.0), "labels must be 0 or 1 for logistic regression loss");
+        var y = targetEnum.Current * 2 - 1; // y in {-1,1}
+
+        yield return weightEnum.Current * 2 * y / (1 + Math.Exp(2 * y * predEnum.Current));
+
       }
       if (targetEnum.MoveNext() | predEnum.MoveNext() | weightEnum.MoveNext())
-        throw new ArgumentException("target, pred and weight have differing lengths");
+        throw new ArgumentException("target, pred and weight have different lengths");
     }
 
@@ -75 +73 @@
 
       if (targetArr.Length != predArr.Length || predArr.Length != weightArr.Length)
-        throw new ArgumentException("target, pred and weight have differing lengths");
+        throw new ArgumentException("target, pred and weight have different lengths");
 
-      // line search for abs error
+      // "Simple Newton-Raphson step" of eqn. 23 
       LineSearchFunc lineSearch = (idx, startIdx, endIdx) => {
         double sumY = 0.0;
         double sumDiff = 0.0;
         for (int i = startIdx; i <= endIdx; i++) {
-          var yi = (targetArr[idx[i]] - predArr[idx[i]]);
-          var wi = weightArr[idx[i]];
+          var row = idx[i];
+          var y = targetArr[row] * 2 - 1; // y in {-1,1}
+          var pseudoResponse = weightArr[row] * 2 * y / (1 + Math.Exp(2 * y * predArr[row]));
 
-          sumY += wi * yi;
-          sumDiff += wi * Math.Abs(yi) * (1 - Math.Abs(yi));
-
+          sumY += pseudoResponse;
+          sumDiff += Math.Abs(pseudoResponse) * (2 - Math.Abs(pseudoResponse));
         }
         // prevent divByZero
         sumDiff = Math.Max(1E-12, sumDiff);
-        return 0.5 * sumY / sumDiff;
+        return sumY / sumDiff;
       };
       return lineSearch;