Changeset 12597

Timestamp:

07/06/15 13:02:19 (9 years ago)

Author:

gkronber

Message:

#2261: comments

Location:

branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees

Files:

• GradientBoostedTreesAlgorithmStatic.cs (modified) (3 diffs)
• LossFunctions/AbsoluteErrorLoss.cs (modified) (2 diffs)
• LossFunctions/RelativeErrorLoss.cs (modified) (3 diffs)
• RegressionTreeBuilder.cs (modified) (1 diff)

branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs

@@ -58 +58 @@
       internal double[] y;
       internal int[] activeIdx;
-      internal double[] rim;
+      internal double[] pseudoRes;
 
       internal IList<IRegressionModel> models;
@@ -88 +88 @@
         pred = Enumerable.Repeat(f0, nRows).ToArray();
         predTest = Enumerable.Repeat(f0, problemData.TestIndices.Count()).ToArray();
-        rim = new double[nRows];
+        pseudoRes = new double[nRows];
 
         models = new List<IRegressionModel>();
@@ -168 +168 @@
       var y = gbmState.y;
       var activeIdx = gbmState.activeIdx;
-      var rim = gbmState.rim;
+      var pseudoRes = gbmState.pseudoRes;
 
       // copy output of gradient function to pre-allocated rim array (pseudo-residuals)
       int rimIdx = 0;
       foreach (var g in lossFunction.GetLossGradient(y, yPred, w)) {
-        rim[rimIdx++] = g;
+        pseudoRes[rimIdx++] = g;
       }
 
-      var tree = treeBuilder.CreateRegressionTreeForGradientBoosting(rim, maxDepth, activeIdx, lossFunction.GetLineSearchFunc(y, yPred, w), r, m);
+      var tree = treeBuilder.CreateRegressionTreeForGradientBoosting(pseudoRes, maxDepth, activeIdx, lossFunction.GetLineSearchFunc(y, yPred, w), r, m);
 
       int i = 0;

branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs

@@ -52 +52 @@
 
       while (targetEnum.MoveNext() & predEnum.MoveNext() & weightEnum.MoveNext()) {
+        // weight * sign(res)
         var res = targetEnum.Current - predEnum.Current;
         if (res > 0) yield return weightEnum.Current;
@@ -61 +62 @@
     }
 
+    // return median of residuals
     public LineSearchFunc GetLineSearchFunc(IEnumerable<double> target, IEnumerable<double> pred, IEnumerable<double> weight) {
       var targetArr = target.ToArray();

branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs

@@ -29 +29 @@
 namespace HeuristicLab.Algorithms.DataAnalysis {
   // relative error loss is a special case of weighted absolute error loss
+  // absolute loss is weighted by (1/target)
   public class RelativeErrorLoss : ILossFunction {
     public double GetLoss(IEnumerable<double> target, IEnumerable<double> pred, IEnumerable<double> weight) {
@@ -52 +53 @@
 
       while (targetEnum.MoveNext() & predEnum.MoveNext() & weightEnum.MoveNext()) {
+        // weight * sign(res) * abs(1 / target)
         var res = targetEnum.Current - predEnum.Current;
         if (res > 0) yield return weightEnum.Current * 1.0 / Math.Abs(targetEnum.Current);
@@ -71 +73 @@
 
       // line search for relative error
-      // TODO: check and improve?
+      // weighted median (weight = 1/target)
       LineSearchFunc lineSearch = (idx, startIdx, endIdx) => {
         // weighted median calculation
         int nRows = endIdx - startIdx + 1; // startIdx and endIdx are inclusive
-        if (nRows == 1) return targetArr[idx[startIdx]] - predArr[idx[startIdx]];
+        if (nRows == 1) return targetArr[idx[startIdx]] - predArr[idx[startIdx]]; // res 
         else if (nRows == 2) {
+          // weighted average of two residuals
           var w0 = weightArr[idx[startIdx]] * Math.Abs(1.0 / targetArr[idx[startIdx]]);
           var w1 = weightArr[idx[endIdx]] * Math.Abs(1.0 / targetArr[idx[endIdx]]);
-          if (w0 > w1) {
-            return targetArr[idx[startIdx]] - predArr[idx[startIdx]];
-          } else if (w0 < w1) {
-            return targetArr[idx[endIdx]] - predArr[idx[endIdx]];
-          } else {
-            // same weight
-            return ((targetArr[idx[startIdx]] - predArr[idx[startIdx]]) + (targetArr[idx[endIdx]] - predArr[idx[endIdx]])) / 2;
-          }
+          return (w0 * (targetArr[idx[startIdx]] - predArr[idx[startIdx]]) + w1 * (targetArr[idx[endIdx]] - predArr[idx[endIdx]])) / (w0 + w1);
         } else {
           var ts = from offset in Enumerable.Range(0, nRows)
                    let i = startIdx + offset
-                   select new { res = targetArr[idx[i]] - predArr[idx[i]], weight = weightArr[idx[i]] * Math.Abs(1.0 / targetArr[idx[i]]) };
+                   let row = idx[i]
+                   select new { res = targetArr[row] - predArr[row], weight = weightArr[row] * Math.Abs(1.0 / targetArr[row]) };
           ts = ts.OrderBy(t => t.res);
           var totalWeight = ts.Sum(t => t.weight);

branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs

@@ -64 +64 @@
     private int curTreeNodeIdx; // the index where the next tree node is stored
 
-    private readonly IList<RegressionTreeModel.TreeNode> nodeQueue; //TODO
+    private readonly IList<RegressionTreeModel.TreeNode> nodeQueue; 
 
     // prepare and allocate buffer variables in ctor