Index: /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj
===================================================================
--- /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj	(revision 17473)
+++ /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj	(revision 17474)
@@ -104,4 +104,8 @@
       <Private>False</Private>
     </Reference>
+    <Reference Include="MathNet.Numerics">
+      <HintPath>..\..\bin\MathNet.Numerics.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core">
@@ -117,4 +121,8 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
+    <Reference Include="TensorFlow.NET.Signed">
+      <HintPath>..\..\bin\TensorFlow.NET.Signed.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
@@ -128,4 +136,5 @@
     <Compile Include="SingleObjective\Evaluators\NonlinearLeastSquaresConstantOptimizationEvaluator.cs" />
     <Compile Include="SingleObjective\Evaluators\SymbolicRegressionMeanRelativeErrorEvaluator.cs" />
+    <Compile Include="SingleObjective\Evaluators\TensorFlowConstantOptimizationEvaluator.cs" />
     <Compile Include="SingleObjective\SymbolicRegressionSolutionsAnalyzer.cs" />
     <Compile Include="SymbolicRegressionPhenotypicDiversityAnalyzer.cs" />
Index: /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs
===================================================================
--- /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs	(revision 17474)
+++ /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs	(revision 17474)
@@ -0,0 +1,139 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Parameters;
+using HEAL.Attic;
+using Tensorflow;
+using static Tensorflow.Binding;
+using DoubleVector = MathNet.Numerics.LinearAlgebra.Vector<double>;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
+  [StorableType("63944BF6-62E5-4BE4-974C-D30AD8770F99")]
+  [Item("TensorFlowConstantOptimizationEvaluator", "")]
+  public class TensorFlowConstantOptimizationEvaluator : SymbolicRegressionConstantOptimizationEvaluator {
+    private const string MaximumIterationsName = "MaximumIterations";
+    private const string LearningRateName = "LearningRate";
+
+    #region Parameter Properties
+    public IFixedValueParameter<IntValue> ConstantOptimizationIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaximumIterationsName]; }
+    }
+    public IFixedValueParameter<DoubleValue> LearningRateParameter {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters[LearningRateName]; }
+    }
+    #endregion
+
+    #region Properties
+    public int ConstantOptimizationIterations {
+      get { return ConstantOptimizationIterationsParameter.Value.Value; }
+    }
+    public double LearningRate {
+      get { return LearningRateParameter.Value.Value; }
+    }
+    #endregion
+
+    public TensorFlowConstantOptimizationEvaluator()
+      : base() {
+      Parameters.Add(new FixedValueParameter<IntValue>(MaximumIterationsName, "Determines how many iterations should be calculated while optimizing the constant of a symbolic expression tree(0 indicates other or default stopping criterion).", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>(LearningRateName, "", new DoubleValue(0.01)));
+    }
+
+    protected TensorFlowConstantOptimizationEvaluator(TensorFlowConstantOptimizationEvaluator original, Cloner cloner)
+      : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new TensorFlowConstantOptimizationEvaluator(this, cloner);
+    }
+
+    [StorableConstructor]
+    protected TensorFlowConstantOptimizationEvaluator(StorableConstructorFlag _) : base(_) { }
+
+    protected override ISymbolicExpressionTree OptimizeConstants(
+      ISymbolicExpressionTree tree, IRegressionProblemData problemData, IEnumerable<int> rows,
+      CancellationToken cancellationToken = default(CancellationToken), EvaluationsCounter counter = null) {
+      return OptimizeTree(tree,
+        problemData, rows,
+        ApplyLinearScalingParameter.ActualValue.Value, UpdateVariableWeights,
+        ConstantOptimizationIterations, LearningRate,
+        cancellationToken, counter);
+    }
+
+    public static ISymbolicExpressionTree OptimizeTree(
+      ISymbolicExpressionTree tree,
+      IRegressionProblemData problemData, IEnumerable<int> rows,
+      bool applyLinearScaling, bool updateVariableWeights, int maxIterations, double learningRate,
+      CancellationToken cancellationToken = default(CancellationToken), EvaluationsCounter counter = null) {
+
+      bool success = TreeToTensorConverter.TryConvert(tree, updateVariableWeights, applyLinearScaling,
+        out Tensor prediction, out Dictionary<TreeToTensorConverter.DataForVariable, Tensor> variables/*, out double[] initialConstants*/);
+
+      var target = tf.placeholder(tf.float64, name: problemData.TargetVariable);
+      int samples = rows.Count();
+      // mse
+      var costs = tf.reduce_sum(tf.square(prediction - target)) / (2.0 * samples);
+      var optimizer = tf.train.GradientDescentOptimizer((float)learningRate);
+
+      // features as feed items
+      var usedVariables = variables.Select(kvp => kvp.Key.variableName).Concat(new[] { problemData.TargetVariable });
+      var variablesFeed = new Hashtable();
+      foreach (var kvp in variables) {
+        var variableName = kvp.Key.variableName;
+        var variable = kvp.Value;
+        if (problemData.Dataset.VariableHasType<double>(variableName))
+          variablesFeed.Add(variable, problemData.Dataset.GetDoubleValues(variableName, rows));
+        if (problemData.Dataset.VariableHasType<string>(variableName))
+          variablesFeed.Add(variable, problemData.Dataset.GetStringValues(variableName, rows));
+        if (problemData.Dataset.VariableHasType<DoubleVector>(variableName))
+          variablesFeed.Add(variable, problemData.Dataset.GetDoubleVectorValues(variableName, rows));
+        throw new NotSupportedException($"Type of the variable is not supported: {variableName}");
+      }
+      variablesFeed.Add(target, problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows));
+
+
+      using (var session = tf.Session()) {
+        for (int i = 0; i < maxIterations; i++) {
+          optimizer.minimize(costs);
+          //var result = session.run(optimizer, variablesFeed);
+        }
+      }
+      optimizer.minimize(costs);
+
+      if (!success)
+        return (ISymbolicExpressionTree)tree.Clone();
+
+
+      return null;
+    }
+
+    public static bool CanOptimizeConstants(ISymbolicExpressionTree tree) {
+      return TreeToTensorConverter.IsCompatible(tree);
+    }
+  }
+}
Index: /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs
===================================================================
--- /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs	(revision 17474)
+++ /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs	(revision 17474)
@@ -0,0 +1,310 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.Serialization;
+using AutoDiff;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using Tensorflow;
+using static Tensorflow.Binding;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
+  public class TreeToTensorConverter {
+
+    #region helper class
+    public class DataForVariable {
+      public readonly string variableName;
+      public readonly string variableValue; // for factor vars
+
+      public DataForVariable(string varName, string varValue) {
+        this.variableName = varName;
+        this.variableValue = varValue;
+      }
+
+      public override bool Equals(object obj) {
+        var other = obj as DataForVariable;
+        if (other == null) return false;
+        return other.variableName.Equals(this.variableName) &&
+               other.variableValue.Equals(this.variableValue);
+      }
+
+      public override int GetHashCode() {
+        return variableName.GetHashCode() ^ variableValue.GetHashCode();
+      }
+    }
+    #endregion
+
+    public static bool TryConvert(ISymbolicExpressionTree tree, bool makeVariableWeightsVariable, bool addLinearScalingTerms,
+      out Tensor graph, out Dictionary<DataForVariable, Tensor> variables/*, out double[] initialConstants*/) {
+
+      try {
+        var converter = new TreeToTensorConverter(makeVariableWeightsVariable, addLinearScalingTerms);
+        graph = converter.ConvertNode(tree.Root.GetSubtree(0));
+
+        //var parametersEntries = converter.parameters.ToList(); // guarantee same order for keys and values
+        variables = converter.parameters; // parametersEntries.Select(kvp => kvp.Value).ToList();
+        //initialConstants = converter.initialConstants.ToArray();
+        return true;
+      } catch (NotSupportedException) {
+        graph = null;
+        variables = null;
+        //initialConstants = null;
+        return false;
+      }
+    }
+
+    private readonly bool makeVariableWeightsVariable;
+    private readonly bool addLinearScalingTerms;
+
+    //private readonly List<double> initialConstants = new List<double>();
+    private readonly Dictionary<DataForVariable, Tensor> parameters = new Dictionary<DataForVariable, Tensor>();
+    private readonly List<Tensor> variables = new List<Tensor>();
+
+    private TreeToTensorConverter(bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
+      this.makeVariableWeightsVariable = makeVariableWeightsVariable;
+      this.addLinearScalingTerms = addLinearScalingTerms;
+    }
+
+
+    private Tensor ConvertNode(ISymbolicExpressionTreeNode node) {
+      if (node.Symbol is Constant) {
+        var value = ((ConstantTreeNode)node).Value;
+        //initialConstants.Add(value);
+        var var = tf.Variable(value);
+        variables.Add(var);
+        return var;
+      }
+
+      if (node.Symbol is Variable || node.Symbol is BinaryFactorVariable) {
+        var varNode = node as VariableTreeNodeBase;
+        var factorVarNode = node as BinaryFactorVariableTreeNode;
+        // factor variable values are only 0 or 1 and set in x accordingly
+        var varValue = factorVarNode != null ? factorVarNode.VariableValue : string.Empty;
+        var par = FindOrCreateParameter(parameters, varNode.VariableName, varValue);
+
+        if (makeVariableWeightsVariable) {
+          //initialConstants.Add(varNode.Weight);
+          var w = tf.Variable(varNode.Weight);
+          variables.Add(w);
+          return w * par;
+        } else {
+          return varNode.Weight * par;
+        }
+      }
+
+      if (node.Symbol is FactorVariable) {
+        var factorVarNode = node as FactorVariableTreeNode;
+        var products = new List<Tensor>();
+        foreach (var variableValue in factorVarNode.Symbol.GetVariableValues(factorVarNode.VariableName)) {
+          var par = FindOrCreateParameter(parameters, factorVarNode.VariableName, variableValue);
+
+          var value = factorVarNode.GetValue(variableValue);
+          //initialConstants.Add(value);
+          var wVar = tf.Variable(value);
+          variables.Add(wVar);
+
+          products.add(wVar * par);
+        }
+
+        return tf.add_n(products.ToArray());
+      }
+
+      if (node.Symbol is Addition) {
+        var terms = new List<Tensor>();
+        foreach (var subTree in node.Subtrees) {
+          terms.Add(ConvertNode(subTree));
+        }
+
+        return tf.add_n(terms.ToArray());
+      }
+
+      if (node.Symbol is Subtraction) {
+        var terms = new List<Tensor>();
+        for (int i = 0; i < node.SubtreeCount; i++) {
+          var t = ConvertNode(node.GetSubtree(i));
+          if (i > 0) t = -t;
+          terms.Add(t);
+        }
+
+        if (terms.Count == 1) return -terms[0];
+        else return tf.add_n(terms.ToArray());
+      }
+
+      if (node.Symbol is Multiplication) {
+        var terms = new List<Tensor>();
+        foreach (var subTree in node.Subtrees) {
+          terms.Add(ConvertNode(subTree));
+        }
+
+        if (terms.Count == 1) return terms[0];
+        else return terms.Aggregate((a, b) => a * b);
+      }
+
+      if (node.Symbol is Division) {
+        var terms = new List<Tensor>();
+        foreach (var subTree in node.Subtrees) {
+          terms.Add(ConvertNode(subTree));
+        }
+
+        if (terms.Count == 1) return 1.0 / terms[0];
+        else return terms.Aggregate((a, b) => a * (1.0 / b));
+      }
+
+      if (node.Symbol is Absolute) {
+        var x1 = ConvertNode(node.GetSubtree(0));
+        return tf.abs(x1);
+      }
+
+      if (node.Symbol is AnalyticQuotient) {
+        var x1 = ConvertNode(node.GetSubtree(0));
+        var x2 = ConvertNode(node.GetSubtree(1));
+        return x1 / tf.pow(1 + x2 * x2, 0.5);
+      }
+
+      if (node.Symbol is Logarithm) {
+        return math_ops.log(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is Exponential) {
+        return math_ops.pow(
+          Math.E,
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is Square) {
+        return tf.square(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is SquareRoot) {
+        return math_ops.sqrt(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is Cube) {
+        return math_ops.pow(
+          ConvertNode(node.GetSubtree(0)), 3.0);
+      }
+
+      if (node.Symbol is CubeRoot) {
+        return math_ops.pow(
+          ConvertNode(node.GetSubtree(0)), 1.0 / 3.0);
+        // TODO
+        // f: x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3),
+        // g:  { var cbrt_x = x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3); return 1.0 / (3 * cbrt_x * cbrt_x); }
+      }
+
+      if (node.Symbol is Sine) {
+        return tf.sin(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is Cosine) {
+        return tf.cos(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is Tangent) {
+        return tf.tan(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is Mean) {
+        return tf.reduce_mean(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      //if (node.Symbol is StandardDeviation) {
+      //  return tf.reduce_std(
+      //    ConvertNode(node.GetSubtree(0)));
+      //}
+
+      if (node.Symbol is Sum) {
+        return tf.reduce_sum(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is StartSymbol) {
+        if (addLinearScalingTerms) {
+          // scaling variables α, β are given at the beginning of the parameter vector
+          var alpha = tf.Variable(1.0);
+          var beta = tf.Variable(0.0);
+          variables.Add(beta);
+          variables.Add(alpha);
+          var t = ConvertNode(node.GetSubtree(0));
+          return t * alpha + beta;
+        } else return ConvertNode(node.GetSubtree(0));
+      }
+
+      throw new NotSupportedException($"Node symbol {node.Symbol} is not supported.");
+    }
+
+    // for each factor variable value we need a parameter which represents a binary indicator for that variable & value combination
+    // each binary indicator is only necessary once. So we only create a parameter if this combination is not yet available
+    private static Tensor FindOrCreateParameter(Dictionary<DataForVariable, Tensor> parameters, string varName, string varValue = "") {
+      var data = new DataForVariable(varName, varValue);
+
+      if (!parameters.TryGetValue(data, out var par)) {
+        // not found -> create new parameter and entries in names and values lists
+        par = tf.placeholder(tf.float64, name: varName);
+        parameters.Add(data, par);
+      }
+      return par;
+    }
+
+    public static bool IsCompatible(ISymbolicExpressionTree tree) {
+      var containsUnknownSymbol = (
+        from n in tree.Root.GetSubtree(0).IterateNodesPrefix()
+        where
+          !(n.Symbol is Variable) &&
+          !(n.Symbol is BinaryFactorVariable) &&
+          !(n.Symbol is FactorVariable) &&
+          !(n.Symbol is Constant) &&
+          !(n.Symbol is Addition) &&
+          !(n.Symbol is Subtraction) &&
+          !(n.Symbol is Multiplication) &&
+          !(n.Symbol is Division) &&
+          !(n.Symbol is Logarithm) &&
+          !(n.Symbol is Exponential) &&
+          !(n.Symbol is SquareRoot) &&
+          !(n.Symbol is Square) &&
+          !(n.Symbol is Sine) &&
+          !(n.Symbol is Cosine) &&
+          !(n.Symbol is Tangent) &&
+          !(n.Symbol is HyperbolicTangent) &&
+          !(n.Symbol is Erf) &&
+          !(n.Symbol is Norm) &&
+          !(n.Symbol is StartSymbol) &&
+          !(n.Symbol is Absolute) &&
+          !(n.Symbol is AnalyticQuotient) &&
+          !(n.Symbol is Cube) &&
+          !(n.Symbol is CubeRoot) &&
+          !(n.Symbol is Mean) &&
+          //!(n.Symbol is StandardDeviation) &&
+          !(n.Symbol is Sum)
+        select n).Any();
+      return !containsUnknownSymbol;
+    }
+  }
+}
Index: /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj
===================================================================
--- /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj	(revision 17473)
+++ /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj	(revision 17474)
@@ -133,4 +133,8 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
+    <Reference Include="TensorFlow.NET.Signed, Version=0.15.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\bin\TensorFlow.NET.Signed.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
@@ -153,4 +157,5 @@
     <Compile Include="Converters\TreeSimplifier.cs" />
     <Compile Include="Converters\DerivativeCalculator.cs" />
+    <Compile Include="Converters\TreeToTensorConverter.cs" />
     <Compile Include="Converters\TreeToAutoDiffTermConverter.cs" />
     <Compile Include="Creators\SymbolicDataAnalysisExpressionBalancedTreeCreator.cs" />
