Index: /branches/3022-FastFunctionExtraction/FFX/BasisFunction.cs
===================================================================
--- /branches/3022-FastFunctionExtraction/FFX/BasisFunction.cs	(revision 17226)
+++ /branches/3022-FastFunctionExtraction/FFX/BasisFunction.cs	(revision 17227)
@@ -1,4 +1,6 @@
-﻿using System;
+﻿using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Text;
@@ -7,12 +9,38 @@
 namespace HeuristicLab.Algorithms.DataAnalysis.FastFunctionExtraction
 {
-    public enum operators { Abs, Log, Sin, Cos };
 
-    class BasisFunction
+    struct BasisFunction
     {
-        public double _val { get; set; }
-        public string _var { get; set; }
-        public double _exp { get; set; }
-        public operators _op { get; set; }
+        public string Var { get; set; }     // e.g. "Abs(Column1 ** 2)"
+        public double[] Val { get; set; }   // this holds the already calculated values, i.e. the function written in Var
+        public bool IsOperator { get; set; }// alg needs to check if basis function has an operator
+        public NonlinOp Operator { get; }
+
+        public BasisFunction(string var, double[] val, bool isOperator, NonlinOp op = NonlinOp.None)
+        {
+            this.Var = var;
+            this.Val= val;
+            this.IsOperator = isOperator;
+            this.Operator = op;
+        }
+
+        public static BasisFunction operator *(BasisFunction a, BasisFunction b)
+        {
+            Debug.Assert(a.Val.Length == b.Val.Length);
+            double[] newVal = new double[a.Val.Length];
+            for(int i = 0; i < a.Val.Length; i++)
+            {
+                newVal[i] = a.Val[i] * b.Val[i];
+            }
+            return new BasisFunction(a.Var + " * " + b.Var, newVal, false);
+        }
+
+        public int Complexity() => 1;
+
+        public ISymbolicExpressionTree Tree()
+        {
+            return null;
+        }
+        
     }
 }
Index: /branches/3022-FastFunctionExtraction/FFX/FastFunctionExtraction.cs
===================================================================
--- /branches/3022-FastFunctionExtraction/FFX/FastFunctionExtraction.cs	(revision 17226)
+++ /branches/3022-FastFunctionExtraction/FFX/FastFunctionExtraction.cs	(revision 17227)
@@ -15,172 +15,450 @@
 using System.Collections.Generic;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
-
-namespace HeuristicLab.Algorithms.DataAnalysis.FastFunctionExtraction
-{
-
-    [Item(Name = "FastFunctionExtraction", Description = "An FFX algorithm.")]
-    [Creatable(Category = CreatableAttribute.Categories.Algorithms, Priority = 999)]
-    [StorableType("689280F7-E371-44A2-98A5-FCEDF22CA343")] // for persistence (storing your algorithm to a files or transfer to HeuristicLab.Hive
-    public sealed class FastFunctionExtraction : FixedDataAnalysisAlgorithm<RegressionProblem>
-    {
-        private enum Operator { Abs, Log };
-        private static readonly double[] exponents = { 0.5, 1, 2 };
-
-        private const string PenaltyParameterName = "Penalty";
-        private const string ConsiderInteractionsParameterName = "Consider Interactions";
-        private const string ConsiderDenominationParameterName = "Consider Denomination";
-        private const string ConsiderExponentiationParameterName = "Consider Exponentiation";
-        private const string ConsiderNonlinearFuncsParameterName = "Consider Nonlinear functions";
-        private const string ConsiderHingeFuncsParameterName = "Consider Hinge Functions";
-
-        #region parameters
-        public IValueParameter<BoolValue> ConsiderInteractionsParameter
-        {
-            get { return (IValueParameter<BoolValue>)Parameters[ConsiderInteractionsParameterName]; }
-        }
-        #endregion
-
-        #region properties
-        public bool ConsiderInteractions
-        {
-            get { return ConsiderInteractionsParameter.Value.Value; }
-            set { ConsiderInteractionsParameter.Value.Value = value; }
-        }
-        #endregion
-
-        [StorableConstructor]
-        private FastFunctionExtraction(StorableConstructorFlag _) : base(_) { }
-        public FastFunctionExtraction(FastFunctionExtraction original, Cloner cloner) : base(original, cloner)
-        {
-            // Don't forget to call the cloning ctor of the base class  
-            // This class does not have fields, therefore we don't need to actually clone anything
-        }
-        public FastFunctionExtraction() : base()
-        {
-            // algorithm parameters are shown in the GUI
-            Parameters.Add(new FixedValueParameter<DoubleValue>(PenaltyParameterName, "Penalty factor (alpha) for balancing between ridge (0.0) and lasso (1.0) regression", new DoubleValue(0.5)));
-            Parameters.Add(new ValueParameter<BoolValue>(ConsiderInteractionsParameterName, "True if you want to consider interactions, otherwise false.", new BoolValue(true)));
-            Parameters.Add(new ValueParameter<BoolValue>(ConsiderDenominationParameterName, "True if you want to consider denominations, otherwise false.", new BoolValue(true)));
-            Parameters.Add(new ValueParameter<BoolValue>(ConsiderExponentiationParameterName, "True if you want to consider exponentiation, otherwise false.", new BoolValue(true)));
-            Parameters.Add(new ValueParameter<BoolValue>(ConsiderNonlinearFuncsParameterName, "True if you want to consider nonlinear functions(abs, log,...), otherwise false.", new BoolValue(true)));
-            Parameters.Add(new ValueParameter<BoolValue>(ConsiderHingeFuncsParameterName, "True if you want to consider Hinge Functions, otherwise false.", new BoolValue(true)));
-        }
-
-        [StorableHook(HookType.AfterDeserialization)]
-        private void AfterDeserialization() { }
-
-        public override IDeepCloneable Clone(Cloner cloner)
-        {
-            return new FastFunctionExtraction(this, cloner);
-        }
-
-        public override Type ProblemType { get { return typeof(RegressionProblem); } }
-        public new RegressionProblem Problem { get { return (RegressionProblem)base.Problem; } }
-
-
-        protected override void Run(CancellationToken cancellationToken)
-        {
-            var basisFunctions = generateBasisFunctions(Problem.ProblemData);
-            var x = Problem.ProblemData.AllowedInputsTrainingValues;
-            List<SymbolicExpressionTree> trees = new List<SymbolicExpressionTree>();
-
-
-            foreach (var basisFunc in basisFunctions)
-            {
-                // add tree representation of basisFunc to trees
-                trees.Add(generateSymbolicExpressionTree(basisFunc));
-            }
-
-            foreach (var tree in trees)
-            {
-                // create new data through the help of the Interpreter
-                //IEnumerable<double> responses = 
-            }
-
-            var coefficientVectorSet = findCoefficientValues(basisFunctions);
-            var paretoFront = nondominatedFilter(coefficientVectorSet);
-        }
-
-        private SymbolicExpressionTree generateSymbolicExpressionTree(KeyValuePair<string, double[]> basisFunc)
-        {
-            throw new NotImplementedException();
-        }
-
-        // generate all possible models
-        private static Dictionary<string, double[]> generateBasisFunctions(IRegressionProblemData problemData)
-        {
-            var basisFunctions = generateUnivariateBases(problemData);
-            return basisFunctions;
-        }
-
-        private static Dictionary<string, double[]> generateUnivariateBases(IRegressionProblemData problemData)
-        {
-
-            var dataset = problemData.Dataset;
-            var rows = problemData.TrainingIndices;
-            var B1 = new Dictionary<string, double[]>();
-
-            foreach (var variableName in dataset.VariableNames)
-            {
-                foreach (var exp in new[] { 0.5, 1, 2 })
-                {
-                    var name = variableName + " ** " + exp;
-                    var data = dataset.GetDoubleValues(variableName, rows).Select(x => Math.Pow(x, exp)).ToArray();
-                    B1.Add(name, data);
-                    foreach (Operator op in Enum.GetValues(typeof(Operator)))
-                    {
-                        var inner_name = op.ToString() + "(" + name + ")";
-                        var inner_data = data.Select(x => executeOperator(x, op)).ToArray();
-                        B1.Add(inner_name, inner_data);
-                    }
-                }
-            }
-            
-            return B1;
-        }
-
-        private static double executeOperator(double x, Operator op)
-        {
-            switch (op)
-            {
-                case Operator.Abs:
-                    return x > 0 ? x : -x;
-                case Operator.Log:
-                    return Math.Log10(x);
-                default:
-                    throw new NotImplementedException();
-            }
-        }
-
-        private static Dictionary<string, double[]> generateMultiVariateBases(Dictionary<string, double[]> B1)
-        {
-            var B2 = new Dictionary<string, double[]>();
-            for(int i = 1; i <= B1.Count(); i++ )
-            {
-                var b_i = B1.ElementAt(i);
-                for (int j = 1; j < i; i++)
-                {
-                    var b_j = B1.ElementAt(j);
-                }
-            }
-
-            // return union of B1 and B2
-            return B2.Concat(B1).ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
-        }
-
-        private static object findCoefficientValues(IEnumerable<KeyValuePair<string, double[]>> basisFunctions)
-        {
-            return new object();
-        }
-
-        private static object nondominatedFilter(object coefficientVectorSet)
-        {
-            return new object();
-        }
-
-        public override bool SupportsPause
-        {
-            get { return false; }
-        }
-    }
+using System.Collections;
+using System.Diagnostics;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
+using HeuristicLab.Analysis;
+using HeuristicLab.Collections;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.FastFunctionExtraction {
+
+  [Item(Name = "FastFunctionExtraction", Description = "An FFX algorithm.")]
+  [Creatable(Category = CreatableAttribute.Categories.Algorithms, Priority = 999)]
+  [StorableType("689280F7-E371-44A2-98A5-FCEDF22CA343")] // for persistence (storing your algorithm to a files or transfer to HeuristicLab.Hive
+  public sealed class FastFunctionExtraction : FixedDataAnalysisAlgorithm<RegressionProblem> {
+
+    private static readonly double[] exponents = { 0.5, 1, 2 };
+    private static readonly OpCode[] nonlinFuncs = { OpCode.Absolute, OpCode.Log, OpCode.Sin, OpCode.Cos };
+
+    private static readonly BidirectionalDictionary<OpCode, string> OpCodeToString = new BidirectionalDictionary<OpCode, string> {
+        { OpCode.Log, "LOG" },
+        { OpCode.Absolute, "ABS"},
+        { OpCode.Sin, "SIN"},
+        { OpCode.Cos, "COS"},
+        { OpCode.Square, "SQR"},
+        { OpCode.SquareRoot, "SQRT"},
+        { OpCode.Cube, "CUBE"},
+        { OpCode.CubeRoot, "CUBEROOT"}
+    };
+
+    private const string ConsiderInteractionsParameterName = "Consider Interactions";
+    private const string ConsiderDenominationParameterName = "Consider Denomination";
+    private const string ConsiderExponentiationParameterName = "Consider Exponentiation";
+    private const string ConsiderNonlinearFuncsParameterName = "Consider Nonlinear functions";
+    private const string ConsiderHingeFuncsParameterName = "Consider Hinge Functions";
+    private const string PenaltyParameterName = "Penalty";
+    private const string LambdaParameterName = "Lambda";
+    private const string NonlinearFuncsParameterName = "Nonlinear Functions";
+
+    #region parameters
+    public IValueParameter<BoolValue> ConsiderInteractionsParameter
+    {
+      get { return (IValueParameter<BoolValue>)Parameters[ConsiderInteractionsParameterName]; }
+    }
+    public IValueParameter<BoolValue> ConsiderDenominationsParameter
+    {
+      get { return (IValueParameter<BoolValue>)Parameters[ConsiderDenominationParameterName]; }
+    }
+    public IValueParameter<BoolValue> ConsiderExponentiationsParameter
+    {
+      get { return (IValueParameter<BoolValue>)Parameters[ConsiderExponentiationParameterName]; }
+    }
+    public IValueParameter<BoolValue> ConsiderNonlinearFuncsParameter
+    {
+      get { return (IValueParameter<BoolValue>)Parameters[ConsiderNonlinearFuncsParameterName]; }
+    }
+    public IValueParameter<BoolValue> ConsiderHingeFuncsParameter
+    {
+      get { return (IValueParameter<BoolValue>)Parameters[ConsiderHingeFuncsParameterName]; }
+    }
+    public IValueParameter<DoubleValue> PenaltyParameter
+    {
+      get { return (IValueParameter<DoubleValue>)Parameters[PenaltyParameterName]; }
+    }
+    public IValueParameter<DoubleValue> LambdaParameter
+    {
+      get { return (IValueParameter<DoubleValue>)Parameters[LambdaParameterName]; }
+    }
+    public IValueParameter<CheckedItemCollection<EnumValue<OpCode>>> NonlinearFuncsParameter
+    {
+      get { return (IValueParameter<CheckedItemCollection<EnumValue<OpCode>>>)Parameters[NonlinearFuncsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public bool ConsiderInteractions
+    {
+      get { return ConsiderInteractionsParameter.Value.Value; }
+      set { ConsiderInteractionsParameter.Value.Value = value; }
+    }
+    public bool ConsiderDenominations
+    {
+      get { return ConsiderDenominationsParameter.Value.Value; }
+      set { ConsiderDenominationsParameter.Value.Value = value; }
+    }
+    public bool ConsiderExponentiations
+    {
+      get { return ConsiderExponentiationsParameter.Value.Value; }
+      set { ConsiderExponentiationsParameter.Value.Value = value; }
+    }
+    public bool ConsiderNonlinearFuncs
+    {
+      get { return ConsiderNonlinearFuncsParameter.Value.Value; }
+      set { ConsiderNonlinearFuncsParameter.Value.Value = value; }
+    }
+    public bool ConsiderHingeFuncs
+    {
+      get { return ConsiderHingeFuncsParameter.Value.Value; }
+      set { ConsiderHingeFuncsParameter.Value.Value = value; }
+    }
+    public double Penalty
+    {
+      get { return PenaltyParameter.Value.Value; }
+      set { PenaltyParameter.Value.Value = value; }
+    }
+    public DoubleValue Lambda
+    {
+      get { return LambdaParameter.Value; }
+      set { LambdaParameter.Value = value; }
+    }
+    public CheckedItemCollection<EnumValue<OpCode>> NonlinearFuncs
+    {
+      get { return NonlinearFuncsParameter.Value; }
+      set { NonlinearFuncsParameter.Value = value; }
+    }
+    #endregion
+
+
+    [StorableConstructor]
+    private FastFunctionExtraction(StorableConstructorFlag _) : base(_) { }
+    public FastFunctionExtraction(FastFunctionExtraction original, Cloner cloner) : base(original, cloner) {
+    }
+    public FastFunctionExtraction() : base() {
+      var items = new CheckedItemCollection<EnumValue<OpCode>>();
+      foreach (var op in nonlinFuncs) {
+        items.Add(new EnumValue<OpCode>(op));
+      }
+      base.Problem = new RegressionProblem();
+      Parameters.Add(new ValueParameter<BoolValue>(ConsiderInteractionsParameterName, "True if you want the models to include interactions, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new ValueParameter<BoolValue>(ConsiderDenominationParameterName, "True if you want the models to include denominations, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new ValueParameter<BoolValue>(ConsiderExponentiationParameterName, "True if you want the models to include exponentiation, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new ValueParameter<BoolValue>(ConsiderNonlinearFuncsParameterName, "True if you want the models to include nonlinear functions(abs, log,...), otherwise false.", new BoolValue(true)));
+      Parameters.Add(new ValueParameter<BoolValue>(ConsiderHingeFuncsParameterName, "True if you want the models to include Hinge Functions, otherwise false.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(PenaltyParameterName, "Penalty factor (alpha) for balancing between ridge (0.0) and lasso (1.0) regression", new DoubleValue(0.9)));
+      Parameters.Add(new OptionalValueParameter<DoubleValue>(LambdaParameterName, "Optional: the value of lambda for which to calculate an elastic-net solution. lambda == null => calculate the whole path of all lambdas"));
+      Parameters.Add(new ValueParameter<CheckedItemCollection<EnumValue<OpCode>>>(NonlinearFuncsParameterName, "What nonlinear functions the models should be able to include.", items));
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new FastFunctionExtraction(this, cloner);
+    }
+
+    public override Type ProblemType { get { return typeof(RegressionProblem); } }
+    public new RegressionProblem Problem { get { return (RegressionProblem)base.Problem; } }
+
+    public override bool SupportsPause { get { return true; } }
+
+    protected override void Run(CancellationToken cancellationToken) {
+      var basisFunctions = createBasisFunctions(Problem.ProblemData);
+      Results.Add(new Result("Basis Functions", "A Dataset consisting of the generated Basis Functions from FFX Alg Step 1.", createProblemData(Problem.ProblemData, basisFunctions)));
+
+      // add denominator bases to the already existing basis functions
+      if (ConsiderDenominations) basisFunctions = basisFunctions.Concat(createDenominatorBases(Problem.ProblemData, basisFunctions)).ToList();
+
+      // create either path of solutions, or one solution for given lambda
+      LearnModels(Problem.ProblemData, basisFunctions);
+    }
+
+    private List<BasisFunction> createBasisFunctions(IRegressionProblemData problemData) {
+      var basisFunctions = createUnivariateBases(problemData);
+      basisFunctions = basisFunctions.Concat(createMultivariateBases(basisFunctions)).ToList();
+      return basisFunctions;
+    }
+
+    private List<BasisFunction> createUnivariateBases(IRegressionProblemData problemData) {
+      var B1 = new List<BasisFunction>();
+      var inputVariables = problemData.AllowedInputVariables;
+      var validExponents = ConsiderExponentiations ? exponents : new double[] { 1 };
+      var validFuncs = NonlinearFuncs.CheckedItems.Select(val => val.Value);
+      // TODO: add Hinge functions
+
+      foreach (var variableName in inputVariables) {
+        foreach (var exp in validExponents) {
+          var data = problemData.Dataset.GetDoubleValues(variableName).Select(x => Math.Pow(x, exp)).ToArray();
+          if (!ok(data)) continue;
+          var name = expToString(exp, variableName);
+          B1.Add(new BasisFunction(name, data, false));
+          foreach (OpCode _op in validFuncs) {
+            var inner_data = data.Select(x => eval(_op, x)).ToArray();
+            if (!ok(inner_data)) continue;
+            var inner_name = OpCodeToString.GetByFirst(_op) + "(" + name + ")";
+            B1.Add(new BasisFunction(inner_name, inner_data, true));
+          }
+        }
+      }
+      return B1;
+    }
+
+    private List<BasisFunction> createMultivariateBases(List<BasisFunction> B1) {
+      if (!ConsiderInteractions) return B1;
+      var B2 = new List<BasisFunction>();
+      for (int i = 0; i < B1.Count(); i++) {
+        var b_i = B1.ElementAt(i);
+        for (int j = 0; j < i; j++) {
+          var b_j = B1.ElementAt(j);
+          if (b_j.IsOperator) continue; // disallow op() * op()
+          var b_inter = b_i * b_j;
+          B2.Add(b_inter);
+        }
+      }
+
+      return B2;
+      // return union of B1 and B2
+    }
+
+    // creates 1 denominator basis function for each corresponding basis function from basisFunctions
+    private IEnumerable<BasisFunction> createDenominatorBases(IRegressionProblemData problemData, IEnumerable<BasisFunction> basisFunctions) {
+      var y = new BasisFunction(problemData.TargetVariable, problemData.TargetVariableValues.ToArray(), false);
+      var denomBasisFuncs = new List<BasisFunction>();
+      foreach (var func in basisFunctions) {
+        var denomFunc = y * func;
+        denomBasisFuncs.Add(denomFunc);
+      }
+      return denomBasisFuncs;
+    }
+
+    private static string expToString(double exponent, string varname) {
+      if (exponent.IsAlmost(1)) return varname;
+      if (exponent.IsAlmost(1 / 2)) return OpCodeToString.GetByFirst(OpCode.SquareRoot) + "(" + varname + ")";
+      if (exponent.IsAlmost(1 / 3)) return OpCodeToString.GetByFirst(OpCode.CubeRoot) + "(" + varname + ")";
+      if (exponent.IsAlmost(2)) return OpCodeToString.GetByFirst(OpCode.Square) + "(" + varname + ")";
+      if (exponent.IsAlmost(3)) return OpCodeToString.GetByFirst(OpCode.Cube) + "(" + varname + ")";
+      else return varname + " ^ " + exponent;
+    }
+
+    public static double eval(OpCode op, double x) {
+      switch (op) {
+        case OpCode.Absolute:
+          return Math.Abs(x);
+        case OpCode.Log:
+          return Math.Log10(x);
+        case OpCode.Sin:
+          return Math.Sin(x);
+        case OpCode.Cos:
+          return Math.Cos(x);
+        default:
+          throw new Exception("Unimplemented operator: " + op.ToString());
+      }
+    }
+
+    private void PathwiseLearning(IRegressionProblemData problemData, List<BasisFunction> basisFunctions) {
+      ElasticNetLinearRegression reg = new ElasticNetLinearRegression();
+      reg.Lambda = Lambda;
+      reg.Penality = Penalty;
+      reg.Problem.ProblemData = createProblemData(problemData, basisFunctions);
+      reg.Start();
+      Results.AddRange(reg.Results);
+    }
+
+    private void LearnModels(IRegressionProblemData problemData, List<BasisFunction> basisFunctions) {
+      double[] lambda;
+      double[] trainNMSE;
+      double[] testNMSE;
+      double[,] coeff;
+      double[] intercept;
+      int numNominatorBases = ConsiderDenominations ? basisFunctions.Count / 2 : basisFunctions.Count;
+
+      // wraps the list of basis functions in a dataset, so that it can be passed on to the ElNet function
+      var X_b = createProblemData(problemData, basisFunctions);
+
+      ElasticNetLinearRegression.RunElasticNetLinearRegression(X_b, Penalty, out lambda, out trainNMSE, out testNMSE, out coeff, out intercept);
+
+      var errorTable = NMSEGraph(coeff, lambda, trainNMSE, testNMSE);
+      Results.Add(new Result(errorTable.Name, errorTable.Description, errorTable));
+      var coeffTable = CoefficientGraph(coeff, lambda, X_b.AllowedInputVariables, X_b.Dataset);
+      Results.Add(new Result(coeffTable.Name, coeffTable.Description, coeffTable));
+
+      ItemCollection<IResult> models = new ItemCollection<IResult>();
+      for (int modelIdx = 0; modelIdx < coeff.GetUpperBound(0); modelIdx++) {
+        var tree = Tree(basisFunctions, GetRow(coeff, modelIdx), intercept[modelIdx]);
+        ISymbolicRegressionModel m = new SymbolicRegressionModel(Problem.ProblemData.TargetVariable, tree, new SymbolicDataAnalysisExpressionTreeInterpreter());
+        ISymbolicRegressionSolution s = new SymbolicRegressionSolution(m, Problem.ProblemData);
+        models.Add(new Result("Solution " + modelIdx, s));
+      }
+
+      Results.Add(new Result("Models", "The model path returned by the Elastic Net Regression (not only the pareto-optimal subset). ", models));
+    }
+
+    private static IndexedDataTable<double> CoefficientGraph(double[,] coeff, double[] lambda, IEnumerable<string> allowedVars, IDataset ds) {
+      var coeffTable = new IndexedDataTable<double>("Coefficients", "The paths of standarized coefficient values over different lambda values");
+      coeffTable.VisualProperties.YAxisMaximumAuto = false;
+      coeffTable.VisualProperties.YAxisMinimumAuto = false;
+      coeffTable.VisualProperties.XAxisMaximumAuto = false;
+      coeffTable.VisualProperties.XAxisMinimumAuto = false;
+
+      coeffTable.VisualProperties.XAxisLogScale = true;
+      coeffTable.VisualProperties.XAxisTitle = "Lambda";
+      coeffTable.VisualProperties.YAxisTitle = "Coefficients";
+      coeffTable.VisualProperties.SecondYAxisTitle = "Number of variables";
+
+      var nLambdas = lambda.Length;
+      var nCoeff = coeff.GetLength(1);
+      var dataRows = new IndexedDataRow<double>[nCoeff];
+      var numNonZeroCoeffs = new int[nLambdas];
+
+      var doubleVariables = allowedVars.Where(ds.VariableHasType<double>);
+      var factorVariableNames = allowedVars.Where(ds.VariableHasType<string>);
+      var factorVariablesAndValues = ds.GetFactorVariableValues(factorVariableNames, Enumerable.Range(0, ds.Rows)); // must consider all factor values (in train and test set)
+      {
+        int i = 0;
+        foreach (var factorVariableAndValues in factorVariablesAndValues) {
+          foreach (var factorValue in factorVariableAndValues.Value) {
+            double sigma = ds.GetStringValues(factorVariableAndValues.Key)
+              .Select(s => s == factorValue ? 1.0 : 0.0)
+              .StandardDeviation(); // calc std dev of binary indicator
+            var path = Enumerable.Range(0, nLambdas).Select(r => Tuple.Create(lambda[r], coeff[r, i] * sigma)).ToArray();
+            dataRows[i] = new IndexedDataRow<double>(factorVariableAndValues.Key + "=" + factorValue, factorVariableAndValues.Key + "=" + factorValue, path);
+            i++;
+          }
+        }
+
+        foreach (var doubleVariable in doubleVariables) {
+          double sigma = ds.GetDoubleValues(doubleVariable).StandardDeviation();
+          var path = Enumerable.Range(0, nLambdas).Select(r => Tuple.Create(lambda[r], coeff[r, i] * sigma)).ToArray();
+          dataRows[i] = new IndexedDataRow<double>(doubleVariable, doubleVariable, path);
+          i++;
+        }
+        // add to coeffTable by total weight (larger area under the curve => more important);
+        foreach (var r in dataRows.OrderByDescending(r => r.Values.Select(t => t.Item2).Sum(x => Math.Abs(x)))) {
+          coeffTable.Rows.Add(r);
+        }
+      }
+
+      for (int i = 0; i < coeff.GetLength(0); i++) {
+        for (int j = 0; j < coeff.GetLength(1); j++) {
+          if (!coeff[i, j].IsAlmost(0.0)) {
+            numNonZeroCoeffs[i]++;
+          }
+        }
+      }
+      if (lambda.Length > 2) {
+        coeffTable.VisualProperties.XAxisMinimumFixedValue = Math.Pow(10, Math.Floor(Math.Log10(lambda.Last())));
+        coeffTable.VisualProperties.XAxisMaximumFixedValue = Math.Pow(10, Math.Ceiling(Math.Log10(lambda.Skip(1).First())));
+      }
+      coeffTable.Rows.Add(new IndexedDataRow<double>("Number of variables", "The number of non-zero coefficients for each step in the path", lambda.Zip(numNonZeroCoeffs, (l, v) => Tuple.Create(l, (double)v))));
+      coeffTable.Rows["Number of variables"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+      coeffTable.Rows["Number of variables"].VisualProperties.SecondYAxis = true;
+
+      return coeffTable;
+    }
+
+    private static IndexedDataTable<double> NMSEGraph(double[,] coeff, double[] lambda, double[] trainNMSE, double[] testNMSE) {
+      var errorTable = new IndexedDataTable<double>("NMSE", "Path of NMSE values over different lambda values");
+      var numNonZeroCoeffs = new int[lambda.Length];
+      errorTable.VisualProperties.YAxisMaximumAuto = false;
+      errorTable.VisualProperties.YAxisMinimumAuto = false;
+      errorTable.VisualProperties.XAxisMaximumAuto = false;
+      errorTable.VisualProperties.XAxisMinimumAuto = false;
+
+      for (int i = 0; i < coeff.GetLength(0); i++) {
+        for (int j = 0; j < coeff.GetLength(1); j++) {
+          if (!coeff[i, j].IsAlmost(0.0)) {
+            numNonZeroCoeffs[i]++;
+          }
+        }
+      }
+
+      errorTable.VisualProperties.YAxisMinimumFixedValue = 0;
+      errorTable.VisualProperties.YAxisMaximumFixedValue = 1.0;
+      errorTable.VisualProperties.XAxisLogScale = true;
+      errorTable.VisualProperties.XAxisTitle = "Lambda";
+      errorTable.VisualProperties.YAxisTitle = "Normalized mean of squared errors (NMSE)";
+      errorTable.VisualProperties.SecondYAxisTitle = "Number of variables";
+      errorTable.Rows.Add(new IndexedDataRow<double>("NMSE (train)", "Path of NMSE values over different lambda values", lambda.Zip(trainNMSE, (l, v) => Tuple.Create(l, v))));
+      errorTable.Rows.Add(new IndexedDataRow<double>("NMSE (test)", "Path of NMSE values over different lambda values", lambda.Zip(testNMSE, (l, v) => Tuple.Create(l, v))));
+      errorTable.Rows.Add(new IndexedDataRow<double>("Number of variables", "The number of non-zero coefficients for each step in the path", lambda.Zip(numNonZeroCoeffs, (l, v) => Tuple.Create(l, (double)v))));
+      if (lambda.Length > 2) {
+        errorTable.VisualProperties.XAxisMinimumFixedValue = Math.Pow(10, Math.Floor(Math.Log10(lambda.Last())));
+        errorTable.VisualProperties.XAxisMaximumFixedValue = Math.Pow(10, Math.Ceiling(Math.Log10(lambda.Skip(1).First())));
+      }
+      errorTable.Rows["NMSE (train)"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+      errorTable.Rows["NMSE (test)"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+      errorTable.Rows["Number of variables"].VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+      errorTable.Rows["Number of variables"].VisualProperties.SecondYAxis = true;
+
+      return errorTable;
+    }
+
+    private ISymbolicExpressionTree Tree(List<BasisFunction> basisFunctions, double[] coeffs, double offset) {
+      Debug.Assert(basisFunctions.Count() == coeffs.Length);
+      //SymbolicExpressionTree
+      var numNumeratorFuncs = ConsiderDenominations ? basisFunctions.Count() / 2 : basisFunctions.Count();
+      var numeratorBasisFuncs = basisFunctions.Take(numNumeratorFuncs);
+
+      // returns true if there exists at least 1 coefficient value in the model that is part of the denominator 
+      // (i.e. if there exists at least 1 non-zero value in the second half of the array)
+      bool withDenom(double[] coeffarr) => coeffarr.Take(coeffarr.Length / 2).ToArray().Any(val => !val.IsAlmost(0.0));
+      string model = "(" + offset.ToString();
+      for (int i = 0; i < numNumeratorFuncs; i++) {
+        var func = basisFunctions.ElementAt(i);
+        // only generate nodes for relevant basis functions (those with non-zero coeffs)
+        if (!coeffs[i].IsAlmost(0.0))
+          model += " + (" + coeffs[i] + ") * " + func.Var;
+      }
+      if (ConsiderDenominations && withDenom(coeffs)) {
+        model += ") / (1";
+        for (int i = numNumeratorFuncs; i < basisFunctions.Count(); i++) {
+          var func = basisFunctions.ElementAt(i);
+          // only generate nodes for relevant basis functions (those with non-zero coeffs)
+          if (!coeffs[i].IsAlmost(0.0))
+            model += " + (" + coeffs[i] + ") * " + func.Var.Substring(4);
+        }
+      }
+      model += ")";
+      InfixExpressionParser p = new InfixExpressionParser();
+      return p.Parse(model);
+    }
+
+    // wraps the list of basis functions into an IRegressionProblemData object
+    private static IRegressionProblemData createProblemData(IRegressionProblemData problemData, List<BasisFunction> basisFunctions) {
+      List<string> variableNames = new List<string>();
+      List<IList> variableVals = new List<IList>();
+      foreach (var basisFunc in basisFunctions) {
+        variableNames.Add(basisFunc.Var);
+        // basisFunctions already contains the calculated values of the corresponding basis function, so you can just take that value 
+        variableVals.Add(new List<double>(basisFunc.Val));
+      }
+      var matrix = new ModifiableDataset(variableNames, variableVals);
+
+      // add the unmodified target variable to the matrix
+      matrix.AddVariable(problemData.TargetVariable, problemData.TargetVariableValues.ToList());
+      var allowedInputVars = matrix.VariableNames.Where(x => !x.Equals(problemData.TargetVariable));
+      IRegressionProblemData rpd = new RegressionProblemData(matrix, allowedInputVars, problemData.TargetVariable);
+      rpd.TrainingPartition.Start = problemData.TrainingPartition.Start;
+      rpd.TrainingPartition.End = problemData.TrainingPartition.End;
+      rpd.TestPartition.Start = problemData.TestPartition.Start;
+      rpd.TestPartition.End = problemData.TestPartition.End;
+      return rpd;
+    }
+
+    private static bool ok(double[] data) => data.All(x => !double.IsNaN(x) && !double.IsInfinity(x));
+
+    // helper function which returns a row of a 2D array
+    private static T[] GetRow<T>(T[,] matrix, int row) {
+      var columns = matrix.GetLength(1);
+      var array = new T[columns];
+      for (int i = 0; i < columns; ++i)
+        array[i] = matrix[row, i];
+      return array;
+    }
+
+    // returns all models with pareto-optimal tradeoff between error and complexity
+    private static List<IRegressionSolution> nondominatedFilter(double[][] coefficientVectorSet, BasisFunction[] basisFunctions) {
+      return null;
+    }
+  }
 }
Index: /branches/3022-FastFunctionExtraction/FFX/FastFunctionExtraction.csproj
===================================================================
--- /branches/3022-FastFunctionExtraction/FFX/FastFunctionExtraction.csproj	(revision 17226)
+++ /branches/3022-FastFunctionExtraction/FFX/FastFunctionExtraction.csproj	(revision 17227)
@@ -43,4 +43,8 @@
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\..\..\trunk\bin\HeuristicLab.Algorithms.DataAnalysis.Glmnet-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Analysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\trunk\bin\HeuristicLab.Analysis-3.3.dll</HintPath>
     </Reference>
     <Reference Include="HeuristicLab.Collections-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
@@ -91,4 +95,8 @@
       <HintPath>..\..\..\trunk\bin\HeuristicLab.Problems.DataAnalysis-3.4.dll</HintPath>
     </Reference>
+    <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\..\trunk\bin\HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.dll</HintPath>
+    </Reference>
     <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -112,5 +120,5 @@
     <Compile Include="BasisFunction.cs" />
     <Compile Include="FastFunctionExtraction.cs" />
-    <Compile Include="GeneralizedLinearModel.cs" />
+    <Compile Include="Operator.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
Index: /branches/3022-FastFunctionExtraction/FFX/GeneralizedLinearModel.cs
===================================================================
--- /branches/3022-FastFunctionExtraction/FFX/GeneralizedLinearModel.cs	(revision 17227)
+++ /branches/3022-FastFunctionExtraction/FFX/GeneralizedLinearModel.cs	(revision 17227)
@@ -0,0 +1,31 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.FastFunctionExtraction
+{
+    // A GLM is a function in the form of a linear combination of basis functions (plus an offset)
+    class GeneralizedLinearModel
+    {
+        public double Offset { get; }
+        public double[] Coefficients { get; }
+        public BasisFunction[] BasisFunctions { get; }
+        public int Complexity() => BasisFunctions.Length + BasisFunctions.Sum(x => x.Complexity());
+
+        public GeneralizedLinearModel(double offset, double[] coefficients, BasisFunction[] basisFunctions)
+        {
+            Offset = offset;
+            Coefficients = coefficients;
+            BasisFunctions = basisFunctions;
+        }
+
+        public double eval()
+        {
+            Debug.Assert(Coefficients.Length == BasisFunctions.Length);
+            return 0;
+        }
+    }
+}
Index: /branches/3022-FastFunctionExtraction/FFX/Operator.cs
===================================================================
--- /branches/3022-FastFunctionExtraction/FFX/Operator.cs	(revision 17227)
+++ /branches/3022-FastFunctionExtraction/FFX/Operator.cs	(revision 17227)
@@ -0,0 +1,15 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.DataAnalysis.FastFunctionExtraction
+{
+    public enum NonlinOp { None, Abs, Log, Sin, Cos };
+    static class Operator
+    {
+        
+        public static string ToString(NonlinOp op, string val) => op != NonlinOp.None ? op.ToString() + "(" + val + ")" : val;
+    }
+}
