Index: /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/AutoDiffConverter.cs
===================================================================
--- /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/AutoDiffConverter.cs	(revision 16507)
+++ /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/AutoDiffConverter.cs	(revision 16507)
@@ -0,0 +1,312 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.Serialization;
+using AutoDiff;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic.ConstantsOptimization{
+  public class AutoDiffConverter {
+
+    /// <summary>
+    /// Converts a symbolic expression tree into a parametetric AutoDiff term.
+    /// </summary>
+    /// <param name="tree">The tree the should be converted.</param>
+    /// <param name="addLinearScalingTerms">A flag that determines whether linear scaling terms should be added to the parametric term.</param>
+    /// <param name="numericNodes">The nodes that contain numeric coefficents that should be added as variables in the term.</param>
+    /// <param name="variableData">The variable information that is used to create parameters in the term.</param>
+    /// <param name="autoDiffTerm">The resulting parametric AutoDiff term.</param>
+    /// <returns>A flag to see if the conversion has succeeded.</returns>
+    public static bool TryConvertToAutoDiff(ISymbolicExpressionTree tree, bool addLinearScalingTerms,
+      IEnumerable<ISymbolicExpressionTreeNode> numericNodes, IEnumerable<VariableData> variableData,
+      out IParametricCompiledTerm autoDiffTerm) {
+      // use a transformator object which holds the state (variable list, parameter list, ...) for recursive transformation of the tree
+      var transformator = new AutoDiffConverter(numericNodes, variableData);
+      AutoDiff.Term term;
+
+      try {
+        term = transformator.ConvertToAutoDiff(tree.Root.GetSubtree(0));
+        if (addLinearScalingTerms) {
+          // scaling variables α, β are given at the end of the parameter vector
+          var alpha = new AutoDiff.Variable();
+          var beta = new AutoDiff.Variable();
+
+          term = term * alpha + beta;
+
+          transformator.variables.Add(alpha);
+          transformator.variables.Add(beta);
+        }
+        var compiledTerm = term.Compile(transformator.variables.ToArray(), transformator.parameters.Values.ToArray());
+        autoDiffTerm = compiledTerm;
+        return true;
+      } catch (ConversionException) {
+        autoDiffTerm = null;
+      }
+      return false;
+    }
+
+    // state for recursive transformation of trees 
+    private readonly HashSet<ISymbolicExpressionTreeNode> nodesForOptimization;
+    private readonly Dictionary<VariableData, AutoDiff.Variable> parameters;
+    private readonly List<AutoDiff.Variable> variables;
+
+    private AutoDiffConverter(IEnumerable<ISymbolicExpressionTreeNode> nodesForOptimization, IEnumerable<VariableData> variableData) {
+      this.nodesForOptimization = new HashSet<ISymbolicExpressionTreeNode>(nodesForOptimization);
+      this.parameters = variableData.ToDictionary(k => k, v => new AutoDiff.Variable());
+      this.variables = new List<AutoDiff.Variable>();
+    }
+
+    private AutoDiff.Term ConvertToAutoDiff(ISymbolicExpressionTreeNode node) {
+      if (node.Symbol is Constant) {
+        var constantNode = node as ConstantTreeNode;
+        var value = constantNode.Value;
+        if (nodesForOptimization.Contains(node)) {
+          AutoDiff.Variable var = new AutoDiff.Variable();
+          variables.Add(var);
+          return var;
+        } else {
+          return value;
+        }
+      }
+      if (node.Symbol is Variable || node.Symbol is BinaryFactorVariable) {
+        var varNode = node as VariableTreeNodeBase;
+        var factorVarNode = node as BinaryFactorVariableTreeNode;
+        // factor variable values are only 0 or 1 and set in x accordingly
+        var varValue = factorVarNode != null ? factorVarNode.VariableValue : string.Empty;
+        var data = new VariableData(varNode.VariableName, varValue, 0);
+        var par = parameters[data];
+        var value = varNode.Weight;
+
+        if (nodesForOptimization.Contains(node)) {
+          AutoDiff.Variable var = new AutoDiff.Variable();
+          variables.Add(var);
+          return AutoDiff.TermBuilder.Product(var, par);
+        } else {
+          return AutoDiff.TermBuilder.Product(value, par);
+        }
+      }
+      if (node.Symbol is FactorVariable) {
+        var factorVarNode = node as FactorVariableTreeNode;
+        var products = new List<Term>();
+        foreach (var variableValue in factorVarNode.Symbol.GetVariableValues(factorVarNode.VariableName)) {
+          var data = new VariableData(factorVarNode.VariableName, variableValue, 0);
+          var par = parameters[data];
+          var value = factorVarNode.GetValue(variableValue);
+
+          if (nodesForOptimization.Contains(node)) {
+            var wVar = new AutoDiff.Variable();
+            variables.Add(wVar);
+
+            products.Add(AutoDiff.TermBuilder.Product(wVar, par));
+          } else {
+            products.Add(AutoDiff.TermBuilder.Product(value, par));
+          }
+        }
+        return AutoDiff.TermBuilder.Sum(products);
+      }
+      if (node.Symbol is LaggedVariable) {
+        var varNode = node as LaggedVariableTreeNode;
+        var data = new VariableData(varNode.VariableName, string.Empty, varNode.Lag);
+        var par = parameters[data];
+        var value = varNode.Weight;
+
+        if (nodesForOptimization.Contains(node)) {
+          AutoDiff.Variable var = new AutoDiff.Variable();
+          variables.Add(var);
+          return AutoDiff.TermBuilder.Product(var, par);
+        } else {
+          return AutoDiff.TermBuilder.Product(value, par);
+        }
+
+      }
+      if (node.Symbol is Addition) {
+        List<AutoDiff.Term> terms = new List<Term>();
+        foreach (var subTree in node.Subtrees) {
+          terms.Add(ConvertToAutoDiff(subTree));
+        }
+        return AutoDiff.TermBuilder.Sum(terms);
+      }
+      if (node.Symbol is Subtraction) {
+        List<AutoDiff.Term> terms = new List<Term>();
+        for (int i = 0; i < node.SubtreeCount; i++) {
+          AutoDiff.Term t = ConvertToAutoDiff(node.GetSubtree(i));
+          if (i > 0) t = -t;
+          terms.Add(t);
+        }
+        if (terms.Count == 1) return -terms[0];
+        else return AutoDiff.TermBuilder.Sum(terms);
+      }
+      if (node.Symbol is Multiplication) {
+        List<AutoDiff.Term> terms = new List<Term>();
+        foreach (var subTree in node.Subtrees) {
+          terms.Add(ConvertToAutoDiff(subTree));
+        }
+        if (terms.Count == 1) return terms[0];
+        else return terms.Aggregate((a, b) => new AutoDiff.Product(a, b));
+      }
+      if (node.Symbol is Division) {
+        List<AutoDiff.Term> terms = new List<Term>();
+        foreach (var subTree in node.Subtrees) {
+          terms.Add(ConvertToAutoDiff(subTree));
+        }
+        if (terms.Count == 1) return 1.0 / terms[0];
+        else return terms.Aggregate((a, b) => new AutoDiff.Product(a, 1.0 / b));
+      }
+      if (node.Symbol is Absolute) {
+        var x1 = ConvertToAutoDiff(node.GetSubtree(0));
+        return abs(x1);
+      }
+      if (node.Symbol is AnalyticQuotient) {
+        var x1 = ConvertToAutoDiff(node.GetSubtree(0));
+        var x2 = ConvertToAutoDiff(node.GetSubtree(1));
+        return x1 / (TermBuilder.Power(1 + x2 * x2, 0.5));
+      }
+      if (node.Symbol is Logarithm) {
+        return AutoDiff.TermBuilder.Log(
+          ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is Exponential) {
+        return AutoDiff.TermBuilder.Exp(
+          ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is Square) {
+        return AutoDiff.TermBuilder.Power(
+          ConvertToAutoDiff(node.GetSubtree(0)), 2.0);
+      }
+      if (node.Symbol is SquareRoot) {
+        return AutoDiff.TermBuilder.Power(
+          ConvertToAutoDiff(node.GetSubtree(0)), 0.5);
+      }
+      if (node.Symbol is Cube) {
+        return AutoDiff.TermBuilder.Power(
+          ConvertToAutoDiff(node.GetSubtree(0)), 3.0);
+      }
+      if (node.Symbol is CubeRoot) {
+        return AutoDiff.TermBuilder.Power(
+          ConvertToAutoDiff(node.GetSubtree(0)), 1.0 / 3.0);
+      }
+      if (node.Symbol is Sine) {
+        return sin(
+          ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is Cosine) {
+        return cos(
+          ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is Tangent) {
+        return tan(
+          ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is Erf) {
+        return erf(
+          ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is Norm) {
+        return norm(
+          ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is StartSymbol) {
+        return ConvertToAutoDiff(node.GetSubtree(0));
+      }
+      throw new ConversionException();
+    }
+
+    #region derivations of functions
+    // create function factory for arctangent
+    private static readonly Func<Term, UnaryFunc> arctan = UnaryFunc.Factory(
+      eval: Math.Atan,
+      diff: x => 1 / (1 + x * x));
+
+    private static readonly Func<Term, UnaryFunc> sin = UnaryFunc.Factory(
+      eval: Math.Sin,
+      diff: Math.Cos);
+
+    private static readonly Func<Term, UnaryFunc> cos = UnaryFunc.Factory(
+      eval: Math.Cos,
+      diff: x => -Math.Sin(x));
+
+    private static readonly Func<Term, UnaryFunc> tan = UnaryFunc.Factory(
+      eval: Math.Tan,
+      diff: x => 1 + Math.Tan(x) * Math.Tan(x));
+
+    private static readonly Func<Term, UnaryFunc> erf = UnaryFunc.Factory(
+      eval: alglib.errorfunction,
+      diff: x => 2.0 * Math.Exp(-(x * x)) / Math.Sqrt(Math.PI));
+
+    private static readonly Func<Term, UnaryFunc> norm = UnaryFunc.Factory(
+      eval: alglib.normaldistribution,
+      diff: x => -(Math.Exp(-(x * x)) * Math.Sqrt(Math.Exp(x * x)) * x) / Math.Sqrt(2 * Math.PI));
+
+    private static readonly Func<Term, UnaryFunc> abs = UnaryFunc.Factory(
+      eval: Math.Abs,
+      diff: x => Math.Sign(x)
+      );
+
+    #endregion
+
+
+    public static bool IsCompatible(ISymbolicExpressionTree tree) {
+      var containsUnknownSymbol = (
+        from n in tree.Root.GetSubtree(0).IterateNodesPrefix()
+        where
+          !(n.Symbol is Variable) &&
+          !(n.Symbol is BinaryFactorVariable) &&
+          !(n.Symbol is FactorVariable) &&
+          !(n.Symbol is LaggedVariable) &&
+          !(n.Symbol is Constant) &&
+          !(n.Symbol is Addition) &&
+          !(n.Symbol is Subtraction) &&
+          !(n.Symbol is Multiplication) &&
+          !(n.Symbol is Division) &&
+          !(n.Symbol is Logarithm) &&
+          !(n.Symbol is Exponential) &&
+          !(n.Symbol is SquareRoot) &&
+          !(n.Symbol is Square) &&
+          !(n.Symbol is Sine) &&
+          !(n.Symbol is Cosine) &&
+          !(n.Symbol is Tangent) &&
+          !(n.Symbol is Erf) &&
+          !(n.Symbol is Norm) &&
+          !(n.Symbol is StartSymbol) &&
+          !(n.Symbol is Absolute) &&
+          !(n.Symbol is AnalyticQuotient) &&
+          !(n.Symbol is Cube) &&
+          !(n.Symbol is CubeRoot)
+        select n).Any();
+      return !containsUnknownSymbol;
+    }
+    #region exception class
+    [Serializable]
+    public class ConversionException : Exception {
+      public ConversionException() { }
+      public ConversionException(string message) : base(message) { }
+      public ConversionException(string message, Exception inner) : base(message, inner) { }
+      protected ConversionException(
+        SerializationInfo info,
+        StreamingContext context) : base(info, context) {
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/IConstantsOptimizer.cs
===================================================================
--- /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/IConstantsOptimizer.cs	(revision 16507)
+++ /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/IConstantsOptimizer.cs	(revision 16507)
@@ -0,0 +1,33 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System.Collections.Generic;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic.ConstantsOptimization {
+  public interface IConstantsOptimizer {
+    bool ApplyLinearScaling { get; set; }
+    int MaximumIterations { get; set; }
+    //bool CountEvaluations { get; set; }
+    void OptimizeConstants(ISymbolicExpressionTree tree, IRegressionProblemData problemData, IEnumerable<int> rows);
+    void OptimizeConstants(ISymbolicExpressionTree tree, IDataset dataset, string targetVariable, IEnumerable<int> rows);
+  }
+}
Index: /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/LMConstantsOptimizer.cs
===================================================================
--- /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/LMConstantsOptimizer.cs	(revision 16507)
+++ /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/LMConstantsOptimizer.cs	(revision 16507)
@@ -0,0 +1,141 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic.ConstantsOptimization {
+  public class LMConstantsOptimizer {
+
+    private LMConstantsOptimizer() { }
+
+    /// <summary>
+    /// Method to determine whether the numeric constants of the tree can be optimized. This depends primarily on the symbols occuring in the tree.
+    /// </summary>
+    /// <param name="tree">The tree that should be analyzed</param>
+    /// <returns>A flag indicating whether the numeric constants of the tree can be optimized</returns>
+    public static bool CanOptimizeConstants(ISymbolicExpressionTree tree) {
+      return AutoDiffConverter.IsCompatible(tree);
+    }
+
+    /// <summary>
+    /// Optimizes the numeric constants in a symbolic expression tree in place.
+    /// </summary>
+    /// <param name="tree">The tree for which the constants should be optimized</param>
+    /// <param name="dataset">The dataset containing the data.</param>
+    /// <param name="targetVariable">The target variable name.</param>
+    /// <param name="rows">The rows for which the data should be extracted.</param>
+    /// <param name="applyLinearScaling">A flag to determine whether linear scaling should be applied during the optimization</param>
+    /// <param name="maxIterations">The maximum number of iterations of the Levenberg-Marquard algorithm.</param>
+    /// <returns></returns>
+    public static double OptimizeConstants(ISymbolicExpressionTree tree,
+      IDataset dataset, string targetVariable, IEnumerable<int> rows,
+      bool applyLinearScaling, int maxIterations = 10) {
+      if (tree == null) throw new ArgumentNullException("tree");
+      if (dataset == null) throw new ArgumentNullException("dataset");
+      if (!dataset.ContainsVariable(targetVariable)) throw new ArgumentException("The dataset does not contain the provided target variable.");
+
+      var allVariables = Util.ExtractVariables(tree);
+      var numericNodes = Util.ExtractNumericNodes(tree);
+
+      AutoDiff.IParametricCompiledTerm term;
+      if (!AutoDiffConverter.TryConvertToAutoDiff(tree, applyLinearScaling, numericNodes, allVariables, out term))
+        throw new NotSupportedException("Could not convert symbolic expression tree to an AutoDiff term due to not supported symbols used in the tree.");
+
+      //Variables of the symbolic expression tree correspond to parameters in the term
+      //Hence if no parameters are present no variables occur in the tree and the R² = 0
+      if (term.Parameters.Count == 0) return 0.0;
+
+      var initialConstants = Util.ExtractConstants(numericNodes, applyLinearScaling);
+      double[] constants;
+      double[,] x = Util.ExtractData(dataset, allVariables, rows);
+      double[] y = dataset.GetDoubleValues(targetVariable, rows).ToArray();
+
+      var result = OptimizeConstants(term, initialConstants, x, y, maxIterations, out constants);
+      if (result != 0.0 && constants.Length != 0)
+        Util.UpdateConstants(numericNodes, constants);
+
+      return result;
+    }
+
+    /// <summary>
+    /// Optimizes the numeric coefficents of an AutoDiff Term using the Levenberg-Marquard algorithm.
+    /// </summary>
+    /// <param name="term">The AutoDiff term for which the numeric coefficients should be optimized.</param>
+    /// <param name="initialConstants">The starting values for the numeric coefficients.</param>
+    /// <param name="x">The input data for the optimization.</param>
+    /// <param name="y">The target values for the optimization.</param>
+    /// <param name="maxIterations">The maximum number of iterations of the Levenberg-Marquard</param>
+    /// <param name="constants">The opitmized constants.</param>
+    /// <param name="LM_IterationCallback">An optional callback for detailed analysis that is called in each algorithm iteration.</param>
+    /// <returns>The R² of the term evaluated on the input data x and the target data y using the optimized constants</returns>
+    public static double OptimizeConstants(AutoDiff.IParametricCompiledTerm term, double[] initialConstants, double[,] x, double[] y,
+      int maxIterations, out double[] constants, Action<double[], double, object> LM_IterationCallback = null) {
+
+      if (term.Parameters.Count == 0) {
+        constants = new double[0];
+        return 0.0;
+      }
+
+      var optimizedConstants = (double[])initialConstants.Clone();
+      int numberOfRows = x.GetLength(0);
+      int numberOfColumns = x.GetLength(1);
+      int numberOfConstants = optimizedConstants.Length;
+
+      alglib.lsfitstate state;
+      alglib.lsfitreport rep;
+      alglib.ndimensional_rep xrep = (p, f, obj) => LM_IterationCallback(p, f, obj);
+      int retVal;
+
+      try {
+        alglib.lsfitcreatefg(x, y, optimizedConstants, numberOfRows, numberOfColumns, numberOfConstants, cheapfg: false, state: out state);
+        alglib.lsfitsetcond(state, 0.0, 0.0, maxIterations);
+        alglib.lsfitsetxrep(state, LM_IterationCallback != null);
+        alglib.lsfitfit(state, Evaluate, EvaluateGradient, xrep, term);
+        alglib.lsfitresults(state, out retVal, out optimizedConstants, out rep);
+      } catch (ArithmeticException) {
+        constants = new double[0];
+        return double.NaN;
+      } catch (alglib.alglibexception) {
+        constants = new double[0];
+        return double.NaN;
+      }
+
+      constants = optimizedConstants;
+      return rep.r2;
+    }
+
+
+    private static void Evaluate(double[] c, double[] x, ref double fx, object o) {
+      AutoDiff.IParametricCompiledTerm term = (AutoDiff.IParametricCompiledTerm)o;
+      fx = term.Evaluate(c, x);
+    }
+
+    private static void EvaluateGradient(double[] c, double[] x, ref double fx, double[] grad, object o) {
+      AutoDiff.IParametricCompiledTerm term = (AutoDiff.IParametricCompiledTerm)o;
+      Tuple<double[], double> result = term.Differentiate(c, x);
+      fx = result.Item2;
+      Array.Copy(result.Item1, grad, grad.Length);
+    }
+  }
+}
Index: /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/Util.cs
===================================================================
--- /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/Util.cs	(revision 16507)
+++ /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/Util.cs	(revision 16507)
@@ -0,0 +1,200 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Common;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic.ConstantsOptimization {
+  public static class Util {
+    /// <summary>
+    /// Extracts all variable information in a symbolic expression tree. The variable information is necessary to convert a tree in an AutoDiff term.
+    /// </summary>
+    /// <param name="tree">The tree referencing the variables.</param>
+    /// <returns>The data for variables occuring in the tree.</returns>
+    public static List<VariableData> ExtractVariables(ISymbolicExpressionTree tree) {
+      if (tree == null) throw new ArgumentNullException("tree");
+
+      var variables = new HashSet<VariableData>();
+      foreach (var node in tree.IterateNodesPrefix().OfType<IVariableTreeNode>()) {
+        string variableName = node.VariableName;
+        int lag = 0;
+        var laggedNode = node as ILaggedTreeNode;
+        if (laggedNode != null) lag = laggedNode.Lag;
+
+
+        var factorNode = node as FactorVariableTreeNode;
+        if (factorNode != null) {
+          foreach (var factorValue in factorNode.Symbol.GetVariableValues(variableName)) {
+            var data = new VariableData(variableName, factorValue, lag);
+            variables.Add(data);
+          }
+        } else {
+          var data = new VariableData(variableName, string.Empty, lag);
+          variables.Add(data);
+        }
+      }
+      return variables.ToList();
+    }
+    /// <summary>
+    /// Extract the necessary date for constants optimization with AutoDiff
+    /// </summary>
+    /// <param name="dataset">The dataset holding the data.</param>
+    /// <param name="variables">The variables for which the data from the dataset should be extracted.</param>
+    /// <param name="rows">The rows for which the data should be extracted.</param>
+    /// <returns>A two-dimensiona double array containing the input data.</returns>
+    public static double[,] ExtractData(IDataset dataset, IEnumerable<VariableData> variables, IEnumerable<int> rows) {
+      if (dataset == null) throw new ArgumentNullException("dataset");
+      if (variables == null) throw new ArgumentNullException("variables");
+      if (rows == null) throw new ArgumentNullException("rows");
+
+      var x = new double[rows.Count(), variables.Count()];
+
+      int col = 0;
+      foreach (var variable in variables) {
+        if (dataset.VariableHasType<double>(variable.variableName)) {
+          IEnumerable<double> values;
+          if (variable.lag == 0)
+            values = dataset.GetDoubleValues(variable.variableName, rows);
+          else
+            values = dataset.GetDoubleValues(variable.variableName, rows.Select(r => r + variable.lag));
+
+          int row = 0;
+          foreach (var value in values) {
+            x[row, col] = value;
+            row++;
+          }
+        } else if (dataset.VariableHasType<string>(variable.variableName)) {
+          var values = dataset.GetStringValues(variable.variableName, rows);
+
+          int row = 0;
+          foreach (var value in values) {
+            x[row, col] = value == variable.variableValue ? 1 : 0; ;
+            row++;
+          }
+        } else throw new NotSupportedException("found a variable of unknown type");
+        col++;
+      }
+
+      return x;
+    }
+
+    /// <summary>
+    /// Extracts all numeric nodes from a symbolic expression tree that can be optimized by the constants optimization
+    /// </summary>
+    /// <param name="tree">The tree from which the numeric nodes should be extracted.</param>
+    /// <returns>A list containing all nodes with numeric coefficients.</returns>
+    public static List<ISymbolicExpressionTreeNode> ExtractNumericNodes(ISymbolicExpressionTree tree) {
+      if (tree == null) throw new ArgumentNullException("tree");
+
+      var nodes = new List<ISymbolicExpressionTreeNode>();
+      foreach (var node in tree.IterateNodesPrefix().OfType<SymbolicExpressionTreeTerminalNode>()) {
+        ConstantTreeNode constantTreeNode = node as ConstantTreeNode;
+        VariableTreeNodeBase variableTreeNodeBase = node as VariableTreeNodeBase;
+        FactorVariableTreeNode factorVarTreeNode = node as FactorVariableTreeNode;
+        if (constantTreeNode != null) nodes.Add(constantTreeNode);
+        else if (variableTreeNodeBase != null) nodes.Add(variableTreeNodeBase);
+        else if (factorVarTreeNode != null) nodes.Add(variableTreeNodeBase);
+        else throw new NotSupportedException(string.Format("Terminal nodes of type {0} are not supported.", node.GetType().GetPrettyName()));
+      }
+      return nodes;
+    }
+
+    /// <summary>
+    /// Extracts all numeric constants from a symbolic expression tree.
+    /// </summary>
+    /// <param name="tree">The tree from which the numeric constants should be extracted.</param>
+    /// <param name="addLinearScalingConstants">Flag to determine whether constants for linear scaling have to be added at the end.
+    /// α *f(x) + β, α = 1.0,  β = 0.0 </param>
+    /// <returns> An array containing the numeric constants.</returns>
+    public static double[] ExtractConstants(ISymbolicExpressionTree tree, bool addLinearScalingConstants) {
+      if (tree == null) throw new ArgumentNullException("tree");
+      return ExtractConstants(tree.IterateNodesPrefix().OfType<SymbolicExpressionTreeTerminalNode>(), addLinearScalingConstants);
+    }
+
+    /// <summary>
+    /// Extracts all numeric constants from a list of nodes.
+    /// </summary>
+    /// <param name="nodes">The list of nodes for which the numeric constants should be extracted.</param>
+    /// <param name="addLinearScalingConstants">Flag to determine whether constants for linear scaling have to be added at the end.
+    /// α *f(x) + β, α = 1.0,  β = 0.0 </param>
+    /// <returns> An array containing the numeric constants.</returns>
+    public static double[] ExtractConstants(IEnumerable<ISymbolicExpressionTreeNode> nodes, bool addLinearScalingConstants) {
+      if (nodes == null) throw new ArgumentNullException("nodes");
+
+      var constants = new List<double>();
+      foreach (var node in nodes) {
+        ConstantTreeNode constantTreeNode = node as ConstantTreeNode;
+        VariableTreeNodeBase variableTreeNodeBase = node as VariableTreeNodeBase;
+        FactorVariableTreeNode factorVarTreeNode = node as FactorVariableTreeNode;
+        if (constantTreeNode != null)
+          constants.Add(constantTreeNode.Value);
+        else if (variableTreeNodeBase != null)
+          constants.Add(variableTreeNodeBase.Weight);
+        else if (factorVarTreeNode != null) {
+          for (int j = 0; j < factorVarTreeNode.Weights.Length; j++)
+            constants.Add(factorVarTreeNode.Weights[j]);
+        } else throw new NotSupportedException(string.Format("Nodes of type {0} are not supported.", node.GetType().GetPrettyName()));
+      }
+      constants.Add(1.0);
+      constants.Add(0.0);
+      return constants.ToArray();
+    }
+
+    /// <summary>
+    /// Sets the numeric constants of the nodes to the provided values.
+    /// </summary>
+    /// <param name="nodes">The nodes whose constants should be updated.</param>
+    /// <param name="constants">The numeric constants which should be set. </param>
+    public static void UpdateConstants(IEnumerable<ISymbolicExpressionTreeNode> nodes, double[] constants) {
+      if (nodes == null) throw new ArgumentNullException("nodes");
+      if (constants == null) throw new ArgumentNullException("constants");
+
+      int i = 0;
+      foreach (var node in nodes) {
+        ConstantTreeNode constantTreeNode = node as ConstantTreeNode;
+        VariableTreeNodeBase variableTreeNodeBase = node as VariableTreeNodeBase;
+        FactorVariableTreeNode factorVarTreeNode = node as FactorVariableTreeNode;
+        if (constantTreeNode != null)
+          constantTreeNode.Value = constants[i++];
+        else if (variableTreeNodeBase != null)
+          variableTreeNodeBase.Weight = constants[i++];
+        else if (factorVarTreeNode != null) {
+          for (int j = 0; j < factorVarTreeNode.Weights.Length; j++)
+            factorVarTreeNode.Weights[j] = constants[i++];
+        } else throw new NotSupportedException(string.Format("Terminal nodes of type {0} are not supported.", node.GetType().GetPrettyName()));
+      }
+    }
+
+    /// <summary>
+    /// Sets all numeric constants of the symbolic expression tree to the provided values.
+    /// </summary>
+    /// <param name="tree">The tree for which the numeric constants should be updated.</param>
+    /// <param name="constants">The numeric constants which should be set.</param>
+    public static void UpdateConstants(ISymbolicExpressionTree tree, double[] constants) {
+      if (tree == null) throw new ArgumentNullException("tree");
+      if (constants == null) throw new ArgumentNullException("constants");
+      UpdateConstants(tree.IterateNodesPrefix().OfType<SymbolicExpressionTreeTerminalNode>(), constants);
+    }
+  }
+}
Index: /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/VariableData.cs
===================================================================
--- /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/VariableData.cs	(revision 16507)
+++ /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/ConstantsOptimization/VariableData.cs	(revision 16507)
@@ -0,0 +1,46 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic.ConstantsOptimization {
+  public class VariableData {
+    public readonly string variableName;
+    public readonly string variableValue; // for factor variables
+    public readonly int lag;
+
+    public VariableData(string varName, string varValue, int lag) {
+      this.variableName = varName;
+      this.variableValue = varValue;
+      this.lag = lag;
+    }
+
+    public override bool Equals(object obj) {
+      var other = obj as VariableData;
+      if (other == null) return false;
+      return other.variableName.Equals(this.variableName) &&
+             other.variableValue.Equals(this.variableValue) &&
+             other.lag == this.lag;
+    }
+
+    public override int GetHashCode() {
+      return variableName.GetHashCode() ^ variableValue.GetHashCode() ^ lag;
+    }
+  }
+}
Index: /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj
===================================================================
--- /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj	(revision 16506)
+++ /branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj	(revision 16507)
@@ -213,7 +213,9 @@
       <SubType>Code</SubType>
     </Compile>
-    <Compile Include="Constants Optimization\IConstantsOptimizer.cs" />
-    <Compile Include="Constants Optimization\LMConstantsOptimizer.cs" />
-    <Compile Include="Constants Optimization\Util.cs" />
+    <Compile Include="ConstantsOptimization\IConstantsOptimizer.cs" />
+    <Compile Include="ConstantsOptimization\LMConstantsOptimizer.cs" />
+    <Compile Include="ConstantsOptimization\AutoDiffConverter.cs" />
+    <Compile Include="ConstantsOptimization\Util.cs" />
+    <Compile Include="ConstantsOptimization\VariableData.cs" />
     <Compile Include="Converters\LinearModelToTreeConverter.cs" />
     <Compile Include="Converters\TreeSimplifier.cs" />
Index: /branches/2974_Constants_Optimization/UnitTests/ConstantsOptimizationTests.cs
===================================================================
--- /branches/2974_Constants_Optimization/UnitTests/ConstantsOptimizationTests.cs	(revision 16506)
+++ /branches/2974_Constants_Optimization/UnitTests/ConstantsOptimizationTests.cs	(revision 16507)
@@ -37,12 +37,13 @@
 
     public static void CompareConstantsOptimizationResults(IRegressionProblemData problemData, ISymbolicExpressionTree tree) {
+      var applyLinearScaling = true;
       var old_optimizedTree = (ISymbolicExpressionTree)tree.Clone();
       var old_result = SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(
         new SymbolicDataAnalysisExpressionTreeLinearInterpreter(),
-        old_optimizedTree, problemData, problemData.TrainingIndices, applyLinearScaling: true, maxIterations: 10);
+        old_optimizedTree, problemData, problemData.TrainingIndices, applyLinearScaling, maxIterations: 10);
 
 
       var new_optimizedTree = (ISymbolicExpressionTree)tree.Clone();
-      var new_result = LMConstantsOptimizer.OptimizeConstants(new_optimizedTree, problemData, problemData.TrainingIndices, applyLinearScaling: true, maxIterations: 10);
+      var new_result = LMConstantsOptimizer.OptimizeConstants(new_optimizedTree, problemData.Dataset, problemData.TargetVariable, problemData.TrainingIndices, applyLinearScaling, maxIterations: 10);
 
       //check R² values
@@ -50,6 +51,6 @@
 
       //check numeric values of constants
-      var old_constants = Util.ExtractConstants(old_optimizedTree);
-      var new_constants = Util.ExtractConstants(new_optimizedTree);
+      var old_constants = Util.ExtractConstants(old_optimizedTree, applyLinearScaling);
+      var new_constants = Util.ExtractConstants(new_optimizedTree, applyLinearScaling);
       //Assert.IsTrue(old_constants.SequenceEqual(new_constants));
 
Index: /branches/2974_Constants_Optimization/UnitTests/PerformanceTest.cs
===================================================================
--- /branches/2974_Constants_Optimization/UnitTests/PerformanceTest.cs	(revision 16506)
+++ /branches/2974_Constants_Optimization/UnitTests/PerformanceTest.cs	(revision 16507)
@@ -6,4 +6,5 @@
 using HeuristicLab.Problems.DataAnalysis;
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Problems.DataAnalysis.Symbolic.ConstantsOptimization;
 using HeuristicLab.Problems.DataAnalysis.Symbolic.Regression;
 using HeuristicLab.Problems.Instances.DataAnalysis;
@@ -23,5 +24,5 @@
     [TestCategory("Problems.DataAnalysis.Symbolic.Regression")]
     [TestProperty("Time", "long")]
-    public void New_ConstantsOptimization_Tower_Algorithm() {
+    public static void New_ConstantsOptimization_Tower_Algorithm() {
       var twister = new MersenneTwister((uint)seed);
       var problemData = new RegressionRealWorldInstanceProvider().LoadData(new Tower());
@@ -40,5 +41,6 @@
       //warm up
       for (int i = 0; i < trees.Length; i++) {
-        double quality = LMConstantsOptimizer.OptimizeConstants(trees[i], problemData, rows, true, maxIterations);
+        if (!trees[i].IterateNodesPrefix().OfType<VariableTreeNode>().Any()) Debugger.Break();
+        double quality = LMConstantsOptimizer.OptimizeConstants(trees[i], problemData.Dataset,problemData.TargetVariable, rows, true, maxIterations);
       }
 
@@ -47,5 +49,5 @@
         watch.Start();
         for (int i = 0; i < trees.Length; i++) {
-          double quality = LMConstantsOptimizer.OptimizeConstants(trees[i], problemData, rows, true, maxIterations);
+          double quality = LMConstantsOptimizer.OptimizeConstants(trees[i], problemData.Dataset, problemData.TargetVariable, rows, true, maxIterations);
         }
         watch.Stop();
@@ -76,4 +78,5 @@
       //warm up
       for (int i = 0; i < trees.Length; i++) {
+        if (!trees[i].IterateNodesPrefix().OfType<VariableTreeNode>().Any()) Debugger.Break();
         double quality = SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(
           interpreter, trees[i], problemData, rows, true, maxIterations);
Index: /branches/2974_Constants_Optimization/UnitTests/UnitTests.csproj
===================================================================
--- /branches/2974_Constants_Optimization/UnitTests/UnitTests.csproj	(revision 16506)
+++ /branches/2974_Constants_Optimization/UnitTests/UnitTests.csproj	(revision 16507)
@@ -11,5 +11,5 @@
     <RootNamespace>UnitTests</RootNamespace>
     <AssemblyName>UnitTests</AssemblyName>
-    <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
@@ -21,4 +21,5 @@
     <NuGetPackageImportStamp>
     </NuGetPackageImportStamp>
+    <TargetFrameworkProfile />
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
