Index: /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/AlglibRbfModel.cs
===================================================================
--- /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/AlglibRbfModel.cs	(revision 17933)
+++ /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/AlglibRbfModel.cs	(revision 17933)
@@ -0,0 +1,101 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HEAL.Attic;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+
+
+  [Item("RBF model (3d)", "Radial-basis-function model (wrapper for alglib.rbfmodel)")]
+  [StorableType("74CE499A-884C-4280-B6BB-DF402692F40D")]
+  public sealed class AlglibRbfModel : RegressionModel {
+    // not storable! see persistence properties below
+    private alglib.rbfmodel model;
+
+    [Storable]
+    private string x1;
+    [Storable]
+    private string x2;
+    [Storable]
+    private string x3;
+    public override IEnumerable<string> VariablesUsedForPrediction => new[] { x1, x2, x3 };
+
+    [StorableConstructor]
+    private AlglibRbfModel(StorableConstructorFlag deserializing) : base(deserializing) {
+      this.model = new alglib.rbfmodel();
+    }
+
+    private AlglibRbfModel(AlglibRbfModel orig, Cloner cloner) : base(orig, cloner) {
+      this.x1 = orig.x1;
+      this.x2 = orig.x2;
+      this.x3 = orig.x3;
+      this.model = (alglib.rbfmodel)orig.model.make_copy();
+    }
+    public AlglibRbfModel(alglib.rbfmodel model, string y, string x1, string x2, string x3)
+      : base(y, $"Spline model ({x1}, {x2}, {x3})") {
+      this.model = (alglib.rbfmodel)model.make_copy();
+      this.x1 = x1;
+      this.x2 = x2;
+      this.x3 = x3;
+    }
+
+
+    public override IDeepCloneable Clone(Cloner cloner) => new AlglibRbfModel(this, cloner);
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      var solution = new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+      solution.Name = $"Alglib RBF model ({x1},{x2},{x3})";
+
+      return solution;
+    }
+
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var x1 = dataset.GetDoubleValues(this.x1, rows).ToArray();
+      var x2 = dataset.GetDoubleValues(this.x2, rows).ToArray();
+      var x3 = dataset.GetDoubleValues(this.x3, rows).ToArray();
+      var res = new double[x1.Length];
+      for (int i = 0; i < x1.Length; i++) {
+        res[i] = alglib.rbfcalc3(model, x1[i], x2[i], x3[i]);
+      }
+      return res;
+    }
+
+    #region persistence
+    [Storable]
+    public string SerializedInterpolant {
+      get {
+        alglib.rbfserialize(model, out var serialized);
+        return serialized;
+      }
+      set {
+        alglib.rbfunserialize(value, out var obj);
+        model = obj;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/GeneralizedAdditiveModelAlgorithm.cs
===================================================================
--- /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/GeneralizedAdditiveModelAlgorithm.cs	(revision 17932)
+++ /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/GeneralizedAdditiveModelAlgorithm.cs	(revision 17933)
@@ -47,4 +47,8 @@
     private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
     private const string CreateSolutionParameterName = "CreateSolution";
+    private const string BaseLearnerParameterName = "BaseLearner";
+    private const string Spline1dBaseLearner = "1d cubic regression spline (penalized)";
+    private const string Spline2dBaseLearner = "2d cubic spline";
+    private const string Spline3dBaseLearner = "3d cubic spline";
 
     #endregion
@@ -72,4 +76,7 @@
     }
 
+    public IConstrainedValueParameter<StringValue> BaseLearnerParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[BaseLearnerParameterName]; }
+    }
     #endregion
 
@@ -101,5 +108,11 @@
     }
 
+    public string BaseLearner {
+      get { return BaseLearnerParameter.Value.Value; }
+      set { BaseLearnerParameter.Value = BaseLearnerParameter.ValidValues.Single(vv => vv.Value == value); }
+    }
     #endregion
+
+    public override bool SupportsStop => true;
 
     [StorableConstructor]
@@ -122,5 +135,5 @@
         "Number of iterations. Try a large value and check convergence of the error over iterations. Usually, only a few iterations (e.g. 10) are needed for convergence.", new IntValue(10)));
       Parameters.Add(new FixedValueParameter<DoubleValue>(LambdaParameterName,
-        "The penalty parameter for the penalized regression splines. Set to a value between -8 (weak smoothing) and 8 (strong smooting). Usually, a value between -4 and 4 should be fine", new DoubleValue(3)));
+        "The penalty parameter for the penalized regression splines. Set to a value between -8 (weak smoothing) and 8 (strong smoothing). Usually, a value between -4 and 4 should be fine", new DoubleValue(3)));
       Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName,
         "The random seed used to initialize the new pseudo random number generator.", new IntValue(0)));
@@ -130,4 +143,6 @@
         "Flag that indicates if a solution should be produced at the end of the run", new BoolValue(true)));
       Parameters[CreateSolutionParameterName].Hidden = true;
+      var validBaseLearners = new ItemSet<StringValue>(new StringValue[] { new StringValue(Spline1dBaseLearner).AsReadOnly(), new StringValue(Spline2dBaseLearner).AsReadOnly(), new StringValue(Spline3dBaseLearner).AsReadOnly()});
+      Parameters.Add(new ConstrainedValueParameter<StringValue>("BaseLearner", "The model to use for the additive functions", validBaseLearners, validBaseLearners.First()));
     }
 
@@ -148,5 +163,12 @@
       var inputVars = problemData.AllowedInputVariables.ToArray();
 
-      int nTerms = inputVars.Length;
+      var maxInteractions = 0;
+      if (BaseLearner == Spline1dBaseLearner) maxInteractions = 1;
+      else if (BaseLearner == Spline2dBaseLearner) maxInteractions = 2;
+      else if (BaseLearner == Spline3dBaseLearner) maxInteractions = 3;
+      else throw new ArgumentException("Unknown base learner.");
+      var varTuples = EnumerableExtensions.Combinations(inputVars, maxInteractions).ToArray();
+      var varTupleNames = varTuples.Select(comb => string.Join(" ", comb)).ToArray();
+      int nTerms = varTuples.Length;
 
       #region init results
@@ -167,5 +189,5 @@
 
       // calculate table with residual contributions of each term
-      var rssTable = new DoubleMatrix(nTerms, 1, new string[] { "RSS" }, inputVars);
+      var rssTable = new DoubleMatrix(nTerms, 1, new string[] { "RSS" }, varTupleNames);
       Results.Add(new Result("RSS Values", rssTable));
       #endregion
@@ -194,5 +216,6 @@
         idx.ShuffleInPlace(rand);
         foreach (var inputIdx in idx) {
-          var inputVar = inputVars[inputIdx];
+          if (cancellationToken.IsCancellationRequested) break;
+          var fInputVars = varTuples[inputIdx];
           // first remove the effect of the previous model for the inputIdx (by adding the output of the current model to the residual)
           AddInPlace(res, f[inputIdx].GetEstimatedValues(ds, trainRows));
@@ -200,5 +223,13 @@
 
           rssTable[inputIdx, 0] = MSE(res);
-          f[inputIdx] = RegressSpline(problemData, inputVar, res, lambda);
+          if (fInputVars.Count() == 1) {
+            f[inputIdx] = Regress1dSpline(problemData, fInputVars.Single(), res, lambda);
+          } else if (fInputVars.Count() == 2) {
+            f[inputIdx] = Regress2dSpline(problemData, fInputVars, res, lambda);
+          } else if (fInputVars.Count() == 3) {
+            f[inputIdx] = Regress3dSpline(problemData, fInputVars, res, lambda);
+          } else {
+            throw new ArgumentException();
+          }
 
           SubtractInPlace(res, f[inputIdx].GetEstimatedValues(ds, trainRows));
@@ -217,5 +248,5 @@
         var model = new RegressionEnsembleModel(f.Concat(new[] { new ConstantModel(avgY, problemData.TargetVariable) }));
         model.AverageModelEstimates = false;
-        var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());        
+        var solution = model.CreateRegressionSolution((IRegressionProblemData)problemData.Clone());
         Results.Add(new Result("Ensemble solution", solution));
       }
@@ -223,5 +254,5 @@
 
     public static double MSE(IEnumerable<double> residuals) {
-      var mse  = residuals.Select(r => r * r).Average();
+      var mse = residuals.Select(r => r * r).Average();
       return mse;
     }
@@ -233,15 +264,50 @@
     }
 
-    private IRegressionModel RegressSpline(IRegressionProblemData problemData, string inputVar, double[] target, double lambda) {
+    private IRegressionModel Regress1dSpline(IRegressionProblemData problemData, string inputVar, double[] target, double lambda) {
       var x = problemData.Dataset.GetDoubleValues(inputVar, problemData.TrainingIndices).ToArray();
       var y = (double[])target.Clone();
-      int info;
       alglib.spline1dinterpolant s;
-      alglib.spline1dfitreport rep;
       int numKnots = (int)Math.Min(50, 3 * Math.Sqrt(x.Length)); // heuristic for number of knots  (Elements of Statistical Learning)
 
-      alglib.spline1dfitpenalized(x, y, numKnots, lambda, out info, out s, out rep);
+      alglib.spline1dfitpenalized(x, y, numKnots, lambda, out _, out s, out _);
 
       return new Spline1dModel(s.innerobj, problemData.TargetVariable, inputVar);
+    }
+
+    private IRegressionModel Regress2dSpline(IRegressionProblemData problemData, IEnumerable<string> inputVars, double[] target, double lambda) {
+      var x = problemData.Dataset.ToArray(inputVars, problemData.TrainingIndices);
+      var xy = x.HorzCat(target);
+
+      alglib.spline2dbuilder builder;
+      int d = 1; // scalar output
+      int numKnots = (int)Math.Min(50, 3 * Math.Sqrt(x.Length)); // heuristic for number of knots  (Elements of Statistical Learning)
+
+      alglib.spline2dbuildercreate(d, out builder);
+      alglib.spline2dbuildersetpoints(builder, xy, xy.GetLength(0));
+      alglib.spline2dbuildersetgrid(builder, numKnots, numKnots);
+      alglib.spline2dbuildersetalgoblocklls(builder, Math.Exp(lambda));
+
+      //
+      // Now we are ready to fit and evaluate our results
+      //
+      alglib.spline2dinterpolant s;
+      alglib.spline2dfit(builder, out s, out _);
+      return new Spline2dModel(s, problemData.TargetVariable, inputVars.ElementAt(0), inputVars.ElementAt(1));
+    }
+
+    private IRegressionModel Regress3dSpline(IRegressionProblemData problemData, IEnumerable<string> inputVars, double[] target, double lambda) {
+      var x = problemData.Dataset.ToArray(inputVars, problemData.TrainingIndices);
+      var xy = x.HorzCat(target);
+
+      var rbase = 100.0;
+      var nlayers = 3;
+
+      alglib.rbfmodel model;
+      alglib.rbfcreate(nx: 3, ny: 1, out model);
+      alglib.rbfsetpoints(model, xy);
+      alglib.rbfsetalgohierarchical(model, rbase, nlayers, Math.Exp(lambda));
+      alglib.rbfbuildmodel(model, out _);
+
+      return new AlglibRbfModel(model, problemData.TargetVariable, inputVars.ElementAt(0), inputVars.ElementAt(1), inputVars.ElementAt(2));
     }
 
Index: /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline1dModel.cs
===================================================================
--- /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline1dModel.cs	(revision 17932)
+++ /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline1dModel.cs	(revision 17933)
@@ -29,4 +29,5 @@
 
 namespace HeuristicLab.Algorithms.DataAnalysis {
+
   [Item("Spline model (1d)",
     "Univariate spline model (wrapper for alglib.spline1dmodel)")]
@@ -40,11 +41,11 @@
       set {
         if (value.Length > 1) throw new ArgumentException("A one-dimensional spline model supports only one input variable.");
-        inputVariable = value[0];
+        x1 = value[0];
       }
     }
 
     [Storable]
-    private string inputVariable;
-    public override IEnumerable<string> VariablesUsedForPrediction => new[] { inputVariable };
+    private string x1;
+    public override IEnumerable<string> VariablesUsedForPrediction => new[] { x1 };
 
     [StorableConstructor]
@@ -54,5 +55,5 @@
 
     private Spline1dModel(Spline1dModel orig, Cloner cloner) : base(orig, cloner) {
-      this.inputVariable = orig.inputVariable;
+      this.x1 = orig.x1;
       this.interpolant = (alglib.spline1d.spline1dinterpolant)orig.interpolant.make_copy();
     }
@@ -60,5 +61,5 @@
       : base(targetVar, $"Spline model ({inputVar})") {
       this.interpolant = (alglib.spline1d.spline1dinterpolant)interpolant.make_copy();
-      this.inputVariable = inputVar;      
+      this.x1 = inputVar;
     }
 
@@ -67,14 +68,14 @@
 
     public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
-      var solution =  new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
-      solution.Name = $"Regression Spline ({inputVariable})";
+      var solution = new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+      solution.Name = $"Regression Spline ({x1})";
 
       return solution;
     }
 
-    public double GetEstimatedValue(double x) => alglib.spline1d.spline1dcalc(interpolant, x);
+    public double GetEstimatedValue(double x) => alglib.spline1d.spline1dcalc(interpolant, x, null);
 
     public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
-      return dataset.GetDoubleValues(inputVariable, rows).Select(GetEstimatedValue);
+      return dataset.GetDoubleValues(x1, rows).Select(GetEstimatedValue);
     }
 
Index: /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline2dModel.cs
===================================================================
--- /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline2dModel.cs	(revision 17933)
+++ /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/GAM/Spline2dModel.cs	(revision 17933)
@@ -0,0 +1,94 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using HEAL.Attic;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Algorithms.DataAnalysis {
+
+
+  [Item("Spline model (2d)",
+    "Bi-variate spline model (wrapper for alglib.spline2sdmodel)")]
+  [StorableType("15E4446C-0C15-4B49-8661-51125071CF81")]
+  public sealed class Spline2dModel : RegressionModel {
+    // not storable! see persistence properties below
+    private alglib.spline2dinterpolant interpolant;
+
+    [Storable]
+    private string x1;
+    [Storable]
+    private string x2;
+    public override IEnumerable<string> VariablesUsedForPrediction => new[] { x1, x2 };
+
+    [StorableConstructor]
+    private Spline2dModel(StorableConstructorFlag deserializing) : base(deserializing) {
+      this.interpolant = new alglib.spline2dinterpolant();
+    }
+
+    private Spline2dModel(Spline2dModel orig, Cloner cloner) : base(orig, cloner) {
+      this.x1 = orig.x1;
+      this.x2 = orig.x2;
+      this.interpolant = (alglib.spline2dinterpolant)orig.interpolant.make_copy();
+    }
+    public Spline2dModel(alglib.spline2dinterpolant interpolant, string y, string x1, string x2)
+      : base(y, $"Spline model ({x1}, {x2})") {
+      this.interpolant = (alglib.spline2dinterpolant)interpolant.make_copy();
+      this.x1 = x1;
+      this.x2 = x2;
+    }
+
+
+    public override IDeepCloneable Clone(Cloner cloner) => new Spline2dModel(this, cloner);
+
+    public override IRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData) {
+      var solution = new RegressionSolution(this, (IRegressionProblemData)problemData.Clone());
+      solution.Name = $"Regression Spline ({x1},{x2})";
+
+      return solution;
+    }
+
+
+    public override IEnumerable<double> GetEstimatedValues(IDataset dataset, IEnumerable<int> rows) {
+      var x1 = dataset.GetDoubleValues(this.x1, rows);
+      var x2 = dataset.GetDoubleValues(this.x2, rows);
+
+      return x1.Zip(x2, (x1i, x2i) => alglib.spline2dcalc(interpolant, x1i, x2i));
+    }
+
+    #region persistence
+    [Storable]
+    public string SerializedInterpolant {
+      get {
+        alglib.spline2dserialize(interpolant, out var serialized);
+        return serialized;
+      }
+      set {
+        alglib.spline2dunserialize(value, out var obj);
+        interpolant = obj;
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj
===================================================================
--- /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 17932)
+++ /branches/3116_GAM_Interactions/HeuristicLab.Algorithms.DataAnalysis/3.4/HeuristicLab.Algorithms.DataAnalysis-3.4.csproj	(revision 17933)
@@ -142,4 +142,6 @@
     <Compile Include="FixedDataAnalysisAlgorithm.cs" />
     <Compile Include="GAM\GeneralizedAdditiveModelAlgorithm.cs" />
+    <Compile Include="GAM\AlglibRbfModel.cs" />
+    <Compile Include="GAM\Spline2dModel.cs" />
     <Compile Include="GAM\Spline1dModel.cs" />
     <Compile Include="GaussianProcess\CovarianceFunctions\CovarianceSpectralMixture.cs" />
