Index: /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs
===================================================================
--- /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 12696)
+++ /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/GradientBoostedTreesAlgorithmStatic.cs	(revision 12697)
@@ -81,6 +81,6 @@
         activeIdx = Enumerable.Range(0, nRows).ToArray();
 
-        var zeros = Enumerable.Repeat(0.0, nRows);
-        double f0 = lossFunction.GetLineSearchFunc(y, zeros)(activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors)
+        var zeros = Enumerable.Repeat(0.0, nRows).ToArray();
+        double f0 = lossFunction.LineSearch(y, zeros, activeIdx, 0, nRows - 1); // initial constant value (mean for squared errors)
         pred = Enumerable.Repeat(f0, nRows).ToArray();
         predTest = Enumerable.Repeat(f0, problemData.TestIndices.Count()).ToArray();
@@ -166,5 +166,5 @@
       }
 
-      var tree = treeBuilder.CreateRegressionTreeForGradientBoosting(pseudoRes, maxSize, activeIdx, lossFunction.GetLineSearchFunc(y, yPred), r, m);
+      var tree = treeBuilder.CreateRegressionTreeForGradientBoosting(pseudoRes, yPred, maxSize, activeIdx, lossFunction, r, m);
 
       int i = 0;
Index: /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs
===================================================================
--- /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs	(revision 12696)
+++ /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/AbsoluteErrorLoss.cs	(revision 12697)
@@ -61,25 +61,18 @@
 
     // return median of residuals
-    public LineSearchFunc GetLineSearchFunc(IEnumerable<double> target, IEnumerable<double> pred) {
-      var targetArr = target.ToArray();
-      var predArr = pred.ToArray();
-
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
       if (targetArr.Length != predArr.Length)
         throw new ArgumentException("target and pred have differing lengths");
 
-      // line search for abs error
-      LineSearchFunc lineSearch = (idx, startIdx, endIdx) => {
-        // Median() is allocating an array anyway
-        // It would be possible to pre-allocated an array for the residuals if Median() would allow specification of a sub-range
-        int nRows = endIdx - startIdx + 1;
-        var res = new double[nRows];
-        for (int i = startIdx; i <= endIdx; i++) {
-          var row = idx[i];
-          res[i - startIdx] = targetArr[row] - predArr[row];
-        }
-        return res.Median(); // TODO: improve efficiency
-      };
-      return lineSearch;
-
+      // Median() is allocating an array anyway
+      // It would be possible to pre-allocated an array for the residuals if Median() would allow specification of a sub-range
+      int nRows = endIdx - startIdx + 1;
+      var res = new double[nRows];
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        res[i - startIdx] = targetArr[row] - predArr[row];
+      }
+      return res.Median(); // TODO: improve efficiency
     }
 
Index: /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs
===================================================================
--- /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs	(revision 12696)
+++ /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/ILossFunction.cs	(revision 12697)
@@ -24,7 +24,4 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
-  // returns the optimal value for the partition of rows stored in idx[startIdx] .. idx[endIdx] inclusive
-  public delegate double LineSearchFunc(int[] idx, int startIdx, int endIdx);
-
   // represents an interface for loss functions used by gradient boosting
   // target represents the target vector  (original targets from the problem data, never changed)
@@ -37,6 +34,6 @@
     IEnumerable<double> GetLossGradient(IEnumerable<double> target, IEnumerable<double> pred);
 
-    // returns a function that returns the optimal prediction value for a subset of rows from target and pred (see LineSearchFunc delegate above)
-    LineSearchFunc GetLineSearchFunc(IEnumerable<double> target, IEnumerable<double> pred);
+    // returns the optimal value for the partition of rows stored in idx[startIdx] .. idx[endIdx] inclusive
+    double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx);
   }
 }
Index: /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs
===================================================================
--- /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs	(revision 12696)
+++ /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/LogisticRegressionLoss.cs	(revision 12697)
@@ -62,29 +62,23 @@
     }
 
-    public LineSearchFunc GetLineSearchFunc(IEnumerable<double> target, IEnumerable<double> pred) {
-      var targetArr = target.ToArray();
-      var predArr = pred.ToArray();
-
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
       if (targetArr.Length != predArr.Length)
         throw new ArgumentException("target and pred have different lengths");
 
       // "Simple Newton-Raphson step" of eqn. 23 
-      LineSearchFunc lineSearch = (idx, startIdx, endIdx) => {
-        double sumY = 0.0;
-        double sumDiff = 0.0;
-        for (int i = startIdx; i <= endIdx; i++) {
-          var row = idx[i];
-          var y = targetArr[row] * 2 - 1; // y in {-1,1}
-          var pseudoResponse = 2 * y / (1 + Math.Exp(2 * y * predArr[row]));
+      double sumY = 0.0;
+      double sumDiff = 0.0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        var row = idx[i];
+        var y = targetArr[row] * 2 - 1; // y in {-1,1}
+        var pseudoResponse = 2 * y / (1 + Math.Exp(2 * y * predArr[row]));
 
-          sumY += pseudoResponse;
-          sumDiff += Math.Abs(pseudoResponse) * (2 - Math.Abs(pseudoResponse));
-        }
-        // prevent divByZero
-        sumDiff = Math.Max(1E-12, sumDiff);
-        return sumY / sumDiff;
-      };
-      return lineSearch;
-
+        sumY += pseudoResponse;
+        sumDiff += Math.Abs(pseudoResponse) * (2 - Math.Abs(pseudoResponse));
+      }
+      // prevent divByZero
+      sumDiff = Math.Max(1E-12, sumDiff);
+      return sumY / sumDiff;
     }
 
Index: /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs
===================================================================
--- /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs	(revision 12696)
+++ /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/RelativeErrorLoss.cs	(revision 12697)
@@ -60,8 +60,6 @@
     }
 
-    public LineSearchFunc GetLineSearchFunc(IEnumerable<double> target, IEnumerable<double> pred) {
-      var targetArr = target.ToArray();
-      var predArr = pred.ToArray();
-
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
       if (targetArr.Length != predArr.Length)
         throw new ArgumentException("target and pred have different lengths");
@@ -69,44 +67,40 @@
       // line search for relative error
       // weighted median (weight = 1/target)
-      LineSearchFunc lineSearch = (idx, startIdx, endIdx) => {
-        // weighted median calculation
-        int nRows = endIdx - startIdx + 1; // startIdx and endIdx are inclusive
-        if (nRows == 1) return targetArr[idx[startIdx]] - predArr[idx[startIdx]]; // res 
-        else if (nRows == 2) {
-          // weighted average of two residuals
-          var w0 = Math.Abs(1.0 / targetArr[idx[startIdx]]);
-          var w1 = Math.Abs(1.0 / targetArr[idx[endIdx]]);
-          if (w0 > w1) {
-            return targetArr[idx[startIdx]] - predArr[idx[startIdx]];
-          } else if (w0 < w1) {
-            return targetArr[idx[endIdx]] - predArr[idx[endIdx]];
-          } else {
-            // same weight -> return average of both residuals
-            return ((targetArr[idx[startIdx]] - predArr[idx[startIdx]]) + (targetArr[idx[endIdx]] - predArr[idx[endIdx]])) / 2;
-          }
+      int nRows = endIdx - startIdx + 1; // startIdx and endIdx are inclusive
+      if (nRows == 1) return targetArr[idx[startIdx]] - predArr[idx[startIdx]]; // res 
+      else if (nRows == 2) {
+        // weighted average of two residuals
+        var w0 = Math.Abs(1.0 / targetArr[idx[startIdx]]);
+        var w1 = Math.Abs(1.0 / targetArr[idx[endIdx]]);
+        if (w0 > w1) {
+          return targetArr[idx[startIdx]] - predArr[idx[startIdx]];
+        } else if (w0 < w1) {
+          return targetArr[idx[endIdx]] - predArr[idx[endIdx]];
         } else {
-          // create an array of key-value pairs to be sorted (instead of using Array.Sort(res, weights))
-          var res_w = new KeyValuePair<double, double>[nRows];
-          var totalWeight = 0.0;
-          for (int i = startIdx; i <= endIdx; i++) {
-            int row = idx[i];
-            var res = targetArr[row] - predArr[row];
-            var w = Math.Abs(1.0 / targetArr[row]);
-            res_w[i - startIdx] = new KeyValuePair<double, double>(res, w);
-            totalWeight += w;
-          }
-          // TODO: improve efficiency (find median without sort)
-          res_w.StableSort((a, b) => Math.Sign(a.Key - b.Key));
+          // same weight -> return average of both residuals
+          return ((targetArr[idx[startIdx]] - predArr[idx[startIdx]]) + (targetArr[idx[endIdx]] - predArr[idx[endIdx]])) / 2;
+        }
+      } else {
+        // create an array of key-value pairs to be sorted (instead of using Array.Sort(res, weights))
+        var res_w = new KeyValuePair<double, double>[nRows];
+        var totalWeight = 0.0;
+        for (int i = startIdx; i <= endIdx; i++) {
+          int row = idx[i];
+          var res = targetArr[row] - predArr[row];
+          var w = Math.Abs(1.0 / targetArr[row]);
+          res_w[i - startIdx] = new KeyValuePair<double, double>(res, w);
+          totalWeight += w;
+        }
+        // TODO: improve efficiency (find median without sort)
+        res_w.StableSort((a, b) => Math.Sign(a.Key - b.Key));
 
-          int k = 0;
-          double sum = totalWeight - res_w[k].Value; // total - first weight
-          while (sum > totalWeight / 2) {
-            k++;
-            sum -= res_w[k].Value;
-          }
-          return res_w[k].Key;
+        int k = 0;
+        double sum = totalWeight - res_w[k].Value; // total - first weight
+        while (sum > totalWeight / 2) {
+          k++;
+          sum -= res_w[k].Value;
         }
-      };
-      return lineSearch;
+        return res_w[k].Key;
+      }
     }
 
Index: /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs
===================================================================
--- /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs	(revision 12696)
+++ /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/LossFunctions/SquaredErrorLoss.cs	(revision 12697)
@@ -53,7 +53,6 @@
     }
 
-    public LineSearchFunc GetLineSearchFunc(IEnumerable<double> target, IEnumerable<double> pred) {
-      var targetArr = target.ToArray();
-      var predArr = pred.ToArray();
+    // targetArr and predArr are not changed by LineSearch
+    public double LineSearch(double[] targetArr, double[] predArr, int[] idx, int startIdx, int endIdx) {
       if (targetArr.Length != predArr.Length)
         throw new ArgumentException("target and pred have different lengths");
@@ -61,16 +60,12 @@
       // line search for squared error loss 
       // for a given partition of rows the optimal constant that should be added to the current prediction values is the average of the residuals
-      LineSearchFunc lineSearch = (idx, startIdx, endIdx) => {
-        double s = 0.0;
-        int n = 0;
-        for (int i = startIdx; i <= endIdx; i++) {
-          int row = idx[i];
-          s += (targetArr[row] - predArr[row]);
-          n++;
-        }
-        return s / n;
-      };
-      return lineSearch;
-
+      double s = 0.0;
+      int n = 0;
+      for (int i = startIdx; i <= endIdx; i++) {
+        int row = idx[i];
+        s += (targetArr[row] - predArr[row]);
+        n++;
+      }
+      return s / n;
     }
 
Index: /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs
===================================================================
--- /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 12696)
+++ /branches/GBT-trunkintegration/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeBuilder.cs	(revision 12697)
@@ -42,4 +42,7 @@
     private readonly int nCols;
     private readonly double[][] x; // all training data (original order from problemData), x is constant
+    private double[] originalY; // the original target labels (from problemData), originalY is constant
+    private double[] curPred; // current predictions for originalY (in case we are using gradient boosting, otherwise = zeros), only necessary for line search
+
     private double[] y; // training labels (original order from problemData), y can be changed
 
@@ -102,6 +105,8 @@
 
       x = new double[nCols][];
-      y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
-
+      originalY = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+      y = new double[originalY.Length];
+      Array.Copy(originalY, y, y.Length); // copy values (originalY is fixed, y is changed in gradient boosting)
+      curPred = Enumerable.Repeat(0.0, y.Length).ToArray(); // zeros
 
       int col = 0;
@@ -127,7 +132,6 @@
 
       var seLoss = new SquaredErrorLoss();
-      var zeros = Enumerable.Repeat(0.0, y.Length);
-
-      var model = CreateRegressionTreeForGradientBoosting(y, maxSize, problemData.TrainingIndices.ToArray(), seLoss.GetLineSearchFunc(y, zeros), r, m);
+
+      var model = CreateRegressionTreeForGradientBoosting(y, curPred, maxSize, problemData.TrainingIndices.ToArray(), seLoss, r, m);
 
       return new GradientBoostedTreesModel(new[] { new ConstantRegressionModel(yAvg), model }, new[] { 1.0, 1.0 });
@@ -135,5 +139,5 @@
 
     // specific interface that allows to specify the target labels and the training rows which is necessary when for gradient boosted trees
-    public IRegressionModel CreateRegressionTreeForGradientBoosting(double[] y, int maxSize, int[] idx, LineSearchFunc lineSearch, double r = 0.5, double m = 0.5) {
+    public IRegressionModel CreateRegressionTreeForGradientBoosting(double[] y, double[] curPred, int maxSize, int[] idx, ILossFunction lossFunction, double r = 0.5, double m = 0.5) {
       Debug.Assert(maxSize > 0);
       Debug.Assert(r > 0);
@@ -143,5 +147,7 @@
       Debug.Assert(m <= 1.0);
 
-      this.y = y; // y is changed in gradient boosting
+      // y and curPred are changed in gradient boosting
+      this.y = y; 
+      this.curPred = curPred; 
 
       // shuffle row idx
@@ -157,5 +163,5 @@
       effectiveVars = (int)Math.Ceiling(nCols * m);
 
-      // the which array is used for partining row idxs 
+      // the which array is used for partitioing row idxs  
       Array.Clear(which, 0, which.Length);
 
@@ -184,8 +190,8 @@
       // and calculate the best split for this root node and enqueue it into a queue sorted by improvement throught the split
       // start and end idx are inclusive
-      CreateLeafNode(0, effectiveRows - 1, lineSearch);
+      CreateLeafNode(0, effectiveRows - 1, lossFunction);
 
       // process the priority queue to complete the tree
-      CreateRegressionTreeFromQueue(maxSize, lineSearch);
+      CreateRegressionTreeFromQueue(maxSize, lossFunction);
 
       return new RegressionTreeModel(tree.ToArray());
@@ -194,5 +200,5 @@
 
     // processes potential splits from the queue as long as splits are left and the maximum size of the tree is not reached
-    private void CreateRegressionTreeFromQueue(int maxNodes, LineSearchFunc lineSearch) {
+    private void CreateRegressionTreeFromQueue(int maxNodes, ILossFunction lossFunction) {
       while (queue.Any() && curTreeNodeIdx + 1 < maxNodes) { // two nodes are created in each loop
         var f = queue[queue.Count - 1]; // last element has the largest improvement
@@ -213,6 +219,6 @@
 
         // create two leaf nodes (and enqueue best splits for both)
-        var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lineSearch);
-        var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lineSearch);
+        var leftTreeIdx = CreateLeafNode(startIdx, splitIdx, lossFunction);
+        var rightTreeIdx = CreateLeafNode(splitIdx + 1, endIdx, lossFunction);
 
         // overwrite existing leaf node with an internal node
@@ -223,7 +229,7 @@
 
     // returns the index of the newly created tree node
-    private int CreateLeafNode(int startIdx, int endIdx, LineSearchFunc lineSearch) {
+    private int CreateLeafNode(int startIdx, int endIdx, ILossFunction lossFunction) {
       // write a leaf node
-      var val = lineSearch(internalIdx, startIdx, endIdx);
+      var val = lossFunction.LineSearch(originalY, curPred, internalIdx, startIdx, endIdx);
       tree[curTreeNodeIdx] = new RegressionTreeModel.TreeNode(RegressionTreeModel.TreeNode.NO_VARIABLE, val);
 
