Index: /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Views/3.4/InteractiveSymbolicRegressionSolutionSimplifierView.cs
===================================================================
--- /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Views/3.4/InteractiveSymbolicRegressionSolutionSimplifierView.cs	(revision 18239)
+++ /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Views/3.4/InteractiveSymbolicRegressionSolutionSimplifierView.cs	(revision 18240)
@@ -97,6 +97,5 @@
       return TensorFlowConstantOptimizationEvaluator.OptimizeTree(tree, regressionProblemData,
         regressionProblemData.TrainingIndices,
-        //new int[]{ 0, 1 },
-        applyLinearScaling: false, updateVariableWeights: true, maxIterations: maxIterations, learningRate: learningRate,
+        applyLinearScaling: true, updateVariableWeights: true, maxIterations: maxIterations, learningRate: learningRate,
         cancellationToken: cancellationToken,
         progress: new SynchronousProgress<double>(cost => {
@@ -126,6 +125,5 @@
           regressionProblemData,
           regressionProblemData.TrainingIndices,
-          //new int[] { 0, 1 },
-          applyLinearScaling: false, maxIterations: constOptIterations, updateVariableWeights: true,
+          applyLinearScaling: true, maxIterations: constOptIterations, updateVariableWeights: true,
           cancellationToken: cancellationToken, iterationCallback: (args, func, obj) => {
             double newProgressValue = progress.ProgressValue + (1.0 / (constOptIterations + 2) / maxRepetitions); // (constOptIterations + 2) iterations are reported
Index: /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs
===================================================================
--- /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs	(revision 18239)
+++ /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs	(revision 18240)
@@ -25,4 +25,5 @@
 
 using System;
+using System.Collections;
 using System.Collections.Generic;
 #if LOG_CONSOLE
@@ -55,5 +56,5 @@
 
     //private static readonly TF_DataType DataType = tf.float64;
-    private static readonly TF_DataType DataType = tf.float32;
+    //private static readonly TF_DataType DataType = tf.float32;
 
     #region Parameter Properties
@@ -108,7 +109,24 @@
       const bool eager = true;
 
-     bool prepared = TreeToTensorConverter.TryPrepareTree(
+#if LOG_FILE
+      var directoryName = $"C:\\temp\\TFboard\\logdir\\TF_{DateTime.Now.ToString("yyyyMMddHHmmss")}_{maxIterations}_{learningRate.ToString(CultureInfo.InvariantCulture)}";
+      Directory.CreateDirectory(directoryName);
+      using var predictionTargetLossWriter = new StreamWriter(File.Create(Path.Combine(directoryName, "PredictionTargetLos.csv")));
+      using var weightsWriter = new StreamWriter(File.Create(Path.Combine(directoryName, "Weights.csv")));
+      using var treeGradsWriter = new StreamWriter(File.Create(Path.Combine(directoryName, "TreeGrads.csv")));
+      using var lossGradsWriter = new StreamWriter(File.Create(Path.Combine(directoryName, "LossGrads.csv")));
+
+      predictionTargetLossWriter.WriteLine(string.Join(";", "Prediction", "Target", "Loss"));
+      weightsWriter.WriteLine(string.Join(";", Enumerable.Range(0, 4).Select(i => $"w_{i}")));
+      treeGradsWriter.WriteLine(string.Join(";", Enumerable.Range(0, 4).Select(i => $"Tg_{i}")));
+      lossGradsWriter.WriteLine(string.Join(";", Enumerable.Range(0, 4).Select(i => $"Lg_{i}")));
+#endif
+
+      //foreach (var row in rows) {
+
+      bool prepared = TreeToTensorConverter.TryPrepareTree(
         tree,
         problemData, rows.ToList(),
+        //problemData, new List<int>(){ row },
         updateVariableWeights, applyLinearScaling,
         eager,
@@ -119,12 +137,16 @@
 
       var optimizer = keras.optimizers.Adam((float)learningRate);
-      
+
       for (int i = 0; i < maxIterations; i++) {
         if (cancellationToken.IsCancellationRequested) break;
 
-        using var tape = tf.GradientTape();
+#if LOG_FILE || LOG_CONSOLE
+        using var tape = tf.GradientTape(persistent: true);
+#else
+        using var tape = tf.GradientTape(persistent: false);
+#endif
 
         bool success = TreeToTensorConverter.TryEvaluate(
-          tree, 
+          tree,
           inputFeatures, variables,
           updateVariableWeights, applyLinearScaling,
@@ -137,11 +159,22 @@
 
         progress?.Report(loss.ToArray<float>()[0]);
-
+        
         var variablesList = variables.Values.SelectMany(x => x).ToList();
         var gradients = tape.gradient(loss, variablesList);
-        
+
+#if LOG_FILE
+        predictionTargetLossWriter.WriteLine(string.Join(";", new[] { prediction.ToArray<float>()[0], target.ToArray<float>()[0], loss.ToArray<float>()[0] }));
+        weightsWriter.WriteLine(string.Join(";", variablesList.Select(v => v.numpy().ToArray<float>()[0])));
+        treeGradsWriter.WriteLine(string.Join(";", tape.gradient(prediction, variablesList).Select(t => t.ToArray<float>()[0])));
+        lossGradsWriter.WriteLine(string.Join(";", tape.gradient(loss, variablesList).Select(t => t.ToArray<float>()[0])));
+#endif
+
+
+        //break;
+
         optimizer.apply_gradients(zip(gradients, variablesList));
       }
-      
+      //}
+
       var cloner = new Cloner();
       var newTree = cloner.Clone(tree);
@@ -153,77 +186,49 @@
 
 
+
+
+      //var numRows = rows.Count();
+      
+      //var variablesFeed = new Hashtable();
+      //foreach (var kvp in inputFeatures) {
+      //  var variableName = kvp.Key;
+      //  var variablePlaceholder = kvp.Value;
+      //  if (problemData.Dataset.VariableHasType<double>(variableName)) {
+      //    var data = problemData.Dataset.GetDoubleValues(variableName, rows).Select(x => (float)x).ToArray();
+      //    variablesFeed.Add(variablePlaceholder, np.array(data).reshape(new Shape(numRows, 1)));
+      //  } else if (problemData.Dataset.VariableHasType<DoubleVector>(variableName)) {
+      //    var data = problemData.Dataset.GetDoubleVectorValues(variableName, rows).SelectMany(x => x.Select(y => (float)y)).ToArray();
+      //    variablesFeed.Add(variablePlaceholder, np.array(data).reshape(new Shape(numRows, -1)));
+      //  } else
+      //    throw new NotSupportedException($"Type of the variable is not supported: {variableName}");
+      //}
+      //var targetData = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows).Select(x => (float)x).ToArray();
+      //variablesFeed.Add(target, np.array(targetData));
+
+      //using var session = tf.Session();
+
+      //var loss2 = tf.constant(1.23f, TF_DataType.TF_FLOAT);
+
+      //var graphOptimizer = tf.train.AdamOptimizer((float)learningRate);
+      //var minimizationOperations = graphOptimizer.minimize(loss2);
+
+      //var init = tf.global_variables_initializer();
+      //session.run(init);
+
+      //session.run((minimizationOperations, loss2), variablesFeed);
+
+      
+      
+
       return newTree;
 
-
-      
-
-
-//      //int numRows = rows.Count();
-
-
-
-
-
-
-//      var variableLengths = problemData.AllowedInputVariables.ToDictionary(
-//        var => var,
-//        var => {
-//          if (problemData.Dataset.VariableHasType<double>(var)) return 1;
-//          if (problemData.Dataset.VariableHasType<DoubleVector>(var)) return problemData.Dataset.GetDoubleVectorValue(var, 0).Count;
-//          throw new NotSupportedException($"Type of variable {var} is not supported.");
-//        });
-
-//      var variablesDict = problemData.AllowedInputVariables.ToDictionary(
-//        var => var,
-//        var => {
-//          if (problemData.Dataset.VariableHasType<double>(var)) {
-//            var data = problemData.Dataset.GetDoubleValues(var, rows).Select(x => (float)x).ToArray();
-//            return tf.convert_to_tensor(np.array(data).reshape(new Shape(numRows, 1)), DataType);
-//          } else if (problemData.Dataset.VariableHasType<DoubleVector>(var)) {
-//            var data = problemData.Dataset.GetDoubleVectorValues(var, rows).SelectMany(x => x.Select(y => (float)y)).ToArray();
-//            return tf.convert_to_tensor(np.array(data).reshape(new Shape(numRows, -1)), DataType);
-//          } else  throw new NotSupportedException($"Type of the variable is not supported: {var}");
-//        }
-//      );
-
-//      using var tape = tf.GradientTape(persistent: true);
-
-//      bool success = TreeToTensorConverter.TryEvaluateEager(tree,
-//        numRows, variablesDict,
-//        updateVariableWeights, applyLinearScaling,
-//        out Tensor prediction,
-//        out Dictionary<Tensor, string> parameters, out List<ResourceVariable> variables);
-
-//      //bool success = TreeToTensorConverter.TryConvert(tree,
-//      //  numRows, variableLengths,
-//      //  updateVariableWeights, applyLinearScaling,
-//      //  out Tensor prediction,
-//      //  out Dictionary<Tensor, string> parameters, out List<Tensor> variables);
-
-//      if (!success)
-//        return (ISymbolicExpressionTree)tree.Clone();
-
-//      //var target = tf.placeholder(DataType, new Shape(numRows), name: problemData.TargetVariable);
-//      var targetData = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows).Select(x => (float)x).ToArray();
-//      var target = tf.convert_to_tensor(np.array(targetData).reshape(new Shape(numRows)), DataType);
-//      // MSE
-//      var cost = tf.reduce_sum(tf.square(prediction - target));
-
-//      tape.watch(cost);
-
-//      //var optimizer = tf.train.AdamOptimizer((float)learningRate);
-//      //var optimizer = tf.train.AdamOptimizer(tf.constant(learningRate, DataType));
-//      //var optimizer = tf.train.GradientDescentOptimizer((float)learningRate);
-//      //var optimizer = tf.train.GradientDescentOptimizer(tf.constant(learningRate, DataType));
-//      //var optimizer = tf.train.GradientDescentOptimizer((float)learningRate);
-//      //var optimizer = tf.train.AdamOptimizer((float)learningRate);
-//      //var optimizationOperation = optimizer.minimize(cost);
-//      var optimizer = keras.optimizers.Adam((float)learningRate);
-
-//      #if EXPORT_GRAPH
+      
+//#if EXPORT_GRAPH
 //      //https://github.com/SciSharp/TensorFlow.NET/wiki/Debugging
 //      tf.train.export_meta_graph(@"C:\temp\TFboard\graph.meta", as_text: false,
 //        clear_devices: true, clear_extraneous_savers: false, strip_default_attrs: true);
 //#endif
+
+      
 
 //      //// features as feed items
@@ -244,120 +249,4 @@
 //      //variablesFeed.Add(target, np.array(targetData));
 
-
-//      List<NDArray> constants;
-//      //using (var session = tf.Session()) {
-
-//#if LOG_FILE
-//        var directoryName = $"C:\\temp\\TFboard\\logdir\\manual_{DateTime.Now.ToString("yyyyMMddHHmmss")}_{maxIterations}_{learningRate.ToString(CultureInfo.InvariantCulture)}";
-//        Directory.CreateDirectory(directoryName);
-//        var costsWriter = new StreamWriter(File.Create(Path.Combine(directoryName, "Costs.csv")));
-//        var weightsWriter = new StreamWriter(File.Create(Path.Combine(directoryName, "Weights.csv")));
-//        var gradientsWriter = new StreamWriter(File.Create(Path.Combine(directoryName, "Gradients.csv")));
-//#endif
-
-//      //session.run(tf.global_variables_initializer());
-
-//#if LOG_CONSOLE || LOG_FILE
-//        var gradients = optimizer.compute_gradients(cost);
-//#endif
-
-//      //var vars = variables.Select(v => session.run(v, variablesFeed)[0].ToArray<float>()[0]).ToList();
-//      //var gradient = optimizer.compute_gradients(cost)
-//      //  .Where(g => g.Item1 != null)
-//      //  //.Select(g => session.run(g.Item1, variablesFeed)[0].GetValue<float>(0)).
-//      //  .Select(g => session.run(g.Item1, variablesFeed)[0].ToArray<float>()[0])
-//      //  .ToList();
-
-//      //var gradientPrediction = optimizer.compute_gradients(prediction)
-//      //  .Where(g => g.Item1 != null)
-//      //  .Select(g => session.run(g.Item1, variablesFeed)[0].ToArray<float>()[0])
-//      //  .ToList();
-
-
-//      //progress?.Report(session.run(cost, variablesFeed)[0].ToArray<float>()[0]);
-//      progress?.Report(cost.ToArray<float>()[0]);
-
-
-      
-
-
-//#if LOG_CONSOLE
-//        Trace.WriteLine("Costs:");
-//        Trace.WriteLine($"MSE: {session.run(cost, variablesFeed)[0].ToString(true)}");
-
-//        Trace.WriteLine("Weights:");
-//        foreach (var v in variables) {
-//          Trace.WriteLine($"{v.name}: {session.run(v).ToString(true)}");
-//        }
-
-//        Trace.WriteLine("Gradients:");
-//        foreach (var t in gradients) {
-//          Trace.WriteLine($"{t.Item2.name}: {session.run(t.Item1, variablesFeed)[0].ToString(true)}");
-//        }
-//#endif
-
-//#if LOG_FILE
-//        costsWriter.WriteLine("MSE");
-//        costsWriter.WriteLine(session.run(cost, variablesFeed)[0].ToArray<float>()[0].ToString(CultureInfo.InvariantCulture));
-
-//        weightsWriter.WriteLine(string.Join(";", variables.Select(v => v.name)));
-//        weightsWriter.WriteLine(string.Join(";", variables.Select(v => session.run(v).ToArray<float>()[0].ToString(CultureInfo.InvariantCulture))));
-
-//        gradientsWriter.WriteLine(string.Join(";", gradients.Select(t => t.Item2.Name)));
-//        gradientsWriter.WriteLine(string.Join(";", gradients.Select(t => session.run(t.Item1, variablesFeed)[0].ToArray<float>()[0].ToString(CultureInfo.InvariantCulture))));
-//#endif
-
-//        for (int i = 0; i < maxIterations; i++) {
-//          if (cancellationToken.IsCancellationRequested)
-//            break;
-
-          
-//        var gradients = tape.gradient(cost, variables);
-//        //optimizer.apply_gradients(gradients.Zip(variables, Tuple.Create<Tensor, IVariableV1>).ToArray());
-//        optimizer.apply_gradients(zip(gradients, variables));
-        
-
-//        //session.run(optimizationOperation, variablesFeed);
-
-//        progress?.Report(cost.ToArray<float>()[0]);
-//        //progress?.Report(session.run(cost, variablesFeed)[0].ToArray<float>()[0]);
-
-//#if LOG_CONSOLE
-//          Trace.WriteLine("Costs:");
-//          Trace.WriteLine($"MSE: {session.run(cost, variablesFeed)[0].ToString(true)}");
-
-//          Trace.WriteLine("Weights:");
-//          foreach (var v in variables) {
-//            Trace.WriteLine($"{v.name}: {session.run(v).ToString(true)}");
-//          }
-
-//          Trace.WriteLine("Gradients:");
-//          foreach (var t in gradients) {
-//            Trace.WriteLine($"{t.Item2.name}: {session.run(t.Item1, variablesFeed)[0].ToString(true)}");
-//          }
-//#endif
-
-//#if LOG_FILE
-//          costsWriter.WriteLine(session.run(cost, variablesFeed)[0].ToArray<float>()[0].ToString(CultureInfo.InvariantCulture));
-//          weightsWriter.WriteLine(string.Join(";", variables.Select(v => session.run(v).ToArray<float>()[0].ToString(CultureInfo.InvariantCulture))));
-//          gradientsWriter.WriteLine(string.Join(";", gradients.Select(t => session.run(t.Item1, variablesFeed)[0].ToArray<float>()[0].ToString(CultureInfo.InvariantCulture))));
-//#endif
-//      }
-
-//#if LOG_FILE
-//        costsWriter.Close();
-//        weightsWriter.Close();
-//        gradientsWriter.Close();
-//#endif
-//      //constants = variables.Select(v => session.run(v)).ToList();
-//      constants = variables.Select(v => v.numpy()).ToList();
-//      //}
-
-//      if (applyLinearScaling)
-//        constants = constants.Skip(2).ToList();
-//      var newTree = (ISymbolicExpressionTree)tree.Clone();
-//      UpdateConstants(newTree, constants, updateVariableWeights);
-
-//      return newTree;
     }
 
@@ -384,18 +273,4 @@
     }
 
-    //private static void UpdateConstants(ISymbolicExpressionTree tree, IList<NDArray> constants, bool updateVariableWeights) {
-    //  int i = 0;
-    //  foreach (var node in tree.Root.IterateNodesPrefix().OfType<SymbolicExpressionTreeTerminalNode>()) {
-    //    if (node is ConstantTreeNode constantTreeNode) {
-    //      constantTreeNode.Value = constants[i++].ToArray<float>()[0];
-    //    } else if (node is VariableTreeNodeBase variableTreeNodeBase && updateVariableWeights) {
-    //      variableTreeNodeBase.Weight = constants[i++].ToArray<float>()[0];
-    //    } else if (node is FactorVariableTreeNode factorVarTreeNode && updateVariableWeights) {
-    //      for (int j = 0; j < factorVarTreeNode.Weights.Length; j++)
-    //        factorVarTreeNode.Weights[j] = constants[i++].ToArray<float>()[0];
-    //    }
-    //  }
-    //}
-
     public static bool CanOptimizeConstants(ISymbolicExpressionTree tree) {
       return TreeToTensorConverter.IsCompatible(tree);
Index: /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs
===================================================================
--- /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs	(revision 18239)
+++ /branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs	(revision 18240)
@@ -58,5 +58,5 @@
 
         inputFeatures = converter.inputFeatures;
-        target = InputFeatureToTensor(problemData.TargetVariable, problemData, rows);
+        target = tf.reshape(InputFeatureToTensor(problemData.TargetVariable, problemData, rows), new Shape(-1));
         variables = converter.variables;
 
@@ -201,7 +201,7 @@
       } else if (node.Symbol is StartSymbol) {
         if (addLinearScalingTerms) {
-          var alpha_arr = np.array(1.0f).reshape(new Shape(1, 1));
+          var alpha_arr = np.array((float)1.0).reshape(new Shape(1, 1));
           var alpha = tf.Variable(alpha_arr, name: "alpha", dtype: DataType);
-          var beta_arr = np.array(0.0f).reshape(new Shape(1, 1));
+          var beta_arr = np.array((float)0.0).reshape(new Shape(1, 1));
           var beta = tf.Variable(beta_arr, name: "beta", dtype: DataType);
           variables.Add(node, new[] { beta, alpha });
@@ -272,6 +272,5 @@
       if (node.Symbol is Division) {
         var terms = node.Subtrees.Select(EvaluateNode).ToList();
-        //if (terms.Count == 1) return 1.0f / terms[0];
-        if (terms.Count == 1) return 1.0 / terms[0];
+        if (terms.Count == 1) return (float)1.0 / terms[0];
         return terms.Aggregate((a, b) => a / b);
       }
@@ -285,5 +284,5 @@
         var x1 = EvaluateNode(node.GetSubtree(0));
         var x2 = EvaluateNode(node.GetSubtree(1));
-        return x1 / tf.pow(1.0f + x2 * x2, 0.5f);
+        return x1 / tf.pow((float)1.0 + x2 * x2, (float)0.5);
         //return x1 / tf.pow(1.0 + x2 * x2, 0.5);
       }
@@ -297,5 +296,4 @@
         return tf.pow(
           (float)Math.E,
-          //Math.E,
           EvaluateNode(node.GetSubtree(0)));
       }
@@ -313,12 +311,10 @@
       if (node.Symbol is Cube) {
         return tf.pow(
-          EvaluateNode(node.GetSubtree(0)), 3.0f);
-        //ConvertNode(node.GetSubtree(0)), 3.0);
+          EvaluateNode(node.GetSubtree(0)), (float)3.0);
       }
 
       if (node.Symbol is CubeRoot) {
         return tf.pow(
-          EvaluateNode(node.GetSubtree(0)), 1.0f / 3.0f);
-        //ConvertNode(node.GetSubtree(0)), 1.0 / 3.0);
+          EvaluateNode(node.GetSubtree(0)), (float)1.0 / (float)3.0);
         // TODO
         // f: x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3),
