Changeset 17476

Timestamp:

03/12/20 17:51:39 (5 years ago)

Author:

pfleck

Message:

#3040 Worked on TF-based constant optimization.

Location:

branches/3040_VectorBasedGP

Files:

• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs (modified) (4 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs (modified) (12 diffs)

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs

@@ -23 +23 @@
 using System.Collections;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading;
@@ -31 +32 @@
 using HeuristicLab.Parameters;
 using HEAL.Attic;
+using NumSharp;
 using Tensorflow;
 using static Tensorflow.Binding;
@@ -92 +94 @@
       CancellationToken cancellationToken = default(CancellationToken), EvaluationsCounter counter = null) {
 
-      bool success = TreeToTensorConverter.TryConvert(tree, updateVariableWeights, applyLinearScaling,
-        out Tensor prediction, out Dictionary<TreeToTensorConverter.DataForVariable, Tensor> variables/*, out double[] initialConstants*/);
+      var vectorVariables = tree.IterateNodesBreadth()
+        .OfType<VariableTreeNodeBase>()
+        .Where(node => problemData.Dataset.VariableHasType<DoubleVector>(node.VariableName))
+        .Select(node => node.VariableName);
+
+      int? vectorLength = null;
+      if (vectorVariables.Any()) {
+        vectorLength = vectorVariables.Select(var => problemData.Dataset.GetDoubleVectorValues(var, rows)).First().First().Count;
+      }
+      int numRows = rows.Count();
+
+      bool success = TreeToTensorConverter.TryConvert(tree,
+        numRows, vectorLength,
+        updateVariableWeights, applyLinearScaling,
+        out Tensor prediction,
+        out Dictionary<Tensor, string> parameters, out List<Tensor> variables/*, out double[] initialConstants*/);
 
       var target = tf.placeholder(tf.float64, name: problemData.TargetVariable);
@@ -99 +115 @@
       // mse
       var costs = tf.reduce_sum(tf.square(prediction - target)) / (2.0 * samples);
-      var optimizer = tf.train.GradientDescentOptimizer((float)learningRate);
+      var optimizer = tf.train.GradientDescentOptimizer((float)learningRate).minimize(costs);
 
       // features as feed items
       var variablesFeed = new Hashtable();
-      foreach (var kvp in variables) {
-        var variableName = kvp.Key.variableName;
-        var variable = kvp.Value;
-        if (problemData.Dataset.VariableHasType<double>(variableName))
-          variablesFeed.Add(variable, problemData.Dataset.GetDoubleValues(variableName, rows));
-        if (problemData.Dataset.VariableHasType<string>(variableName))
-          variablesFeed.Add(variable, problemData.Dataset.GetStringValues(variableName, rows));
-        if (problemData.Dataset.VariableHasType<DoubleVector>(variableName))
-          variablesFeed.Add(variable, problemData.Dataset.GetDoubleVectorValues(variableName, rows));
-        throw new NotSupportedException($"Type of the variable is not supported: {variableName}");
+      foreach (var kvp in parameters) {
+        var variable = kvp.Key;
+        var variableName = kvp.Value;
+        if (problemData.Dataset.VariableHasType<double>(variableName)) {
+          var data = problemData.Dataset.GetDoubleValues(variableName, rows).ToArray();
+          if (vectorLength.HasValue) {
+            var vectorData = new double[numRows][];
+            for (int i = 0; i < numRows; i++)
+              vectorData[i] = Enumerable.Repeat(data[i], vectorLength.Value).ToArray();
+            variablesFeed.Add(variable, np.array(vectorData));
+          } else
+            variablesFeed.Add(variable, np.array(data, copy: false));
+          //} else if (problemData.Dataset.VariableHasType<string>(variableName)) {
+          //  variablesFeed.Add(variable, problemData.Dataset.GetStringValues(variableName, rows));
+        } else if (problemData.Dataset.VariableHasType<DoubleVector>(variableName)) {
+          var data = problemData.Dataset.GetDoubleVectorValues(variableName, rows).Select(x => x.ToArray()).ToArray();
+          variablesFeed.Add(variable, np.array(data));
+        } else
+          throw new NotSupportedException($"Type of the variable is not supported: {variableName}");
       }
-      variablesFeed.Add(target, problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows));
+      var targetData = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows).ToArray();
+      variablesFeed.Add(target, np.array(targetData, copy: false));
 
 
       using (var session = tf.Session()) {
+        session.run(tf.global_variables_initializer());
+
+        Trace.WriteLine("Weights:");
+        foreach (var v in variables)
+          Trace.WriteLine($"{v.name}: {session.run(v).ToString(true)}");
+
         for (int i = 0; i < maxIterations; i++) {
-          optimizer.minimize(costs);
-          var result = session.run(optimizer, variablesFeed);
+
+          //optimizer.minimize(costs);
+          session.run(optimizer, variablesFeed);
+
+          Trace.WriteLine("Weights:");
+          foreach (var v in variables)
+            Trace.WriteLine($"{v.name}: {session.run(v).ToString(true)}");
         }
       }
-      optimizer.minimize(costs);
 
       if (!success)

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs

@@ -23 +23 @@
 using System.Collections.Generic;
 using System.Linq;
-using System.Runtime.Serialization;
-using AutoDiff;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
 using Tensorflow;
 using static Tensorflow.Binding;
+using DoubleVector = MathNet.Numerics.LinearAlgebra.Vector<double>;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
@@ -55 +54 @@
     #endregion
 
-    public static bool TryConvert(ISymbolicExpressionTree tree, bool makeVariableWeightsVariable, bool addLinearScalingTerms,
-      out Tensor graph, out Dictionary<DataForVariable, Tensor> variables/*, out double[] initialConstants*/) {
+    public static bool TryConvert(ISymbolicExpressionTree tree, int numRows, int? vectorLength,
+      bool makeVariableWeightsVariable, bool addLinearScalingTerms,
+      out Tensor graph, out Dictionary<Tensor, string> parameters, out List<Tensor> variables
+/*, out double[] initialConstants*/) {
 
       try {
-        var converter = new TreeToTensorConverter(makeVariableWeightsVariable, addLinearScalingTerms);
+        var converter = new TreeToTensorConverter(numRows, vectorLength, makeVariableWeightsVariable, addLinearScalingTerms);
         graph = converter.ConvertNode(tree.Root.GetSubtree(0));
 
         //var parametersEntries = converter.parameters.ToList(); // guarantee same order for keys and values
-        variables = converter.parameters; // parametersEntries.Select(kvp => kvp.Value).ToList();
+        parameters = converter.parameters; // parametersEntries.Select(kvp => kvp.Value).ToList();
+        variables = converter.variables;
         //initialConstants = converter.initialConstants.ToArray();
         return true;
       } catch (NotSupportedException) {
         graph = null;
+        parameters = null;
         variables = null;
         //initialConstants = null;
@@ -74 +77 @@
     }
 
+    private readonly int numRows;
+    private readonly int? vectorLength;
     private readonly bool makeVariableWeightsVariable;
     private readonly bool addLinearScalingTerms;
 
     //private readonly List<double> initialConstants = new List<double>();
-    private readonly Dictionary<DataForVariable, Tensor> parameters = new Dictionary<DataForVariable, Tensor>();
+    private readonly Dictionary<Tensor, string> parameters = new Dictionary<Tensor, string>();
     private readonly List<Tensor> variables = new List<Tensor>();
 
-    private TreeToTensorConverter(bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
+    private TreeToTensorConverter(int numRows, int? vectorLength, bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
+      this.numRows = numRows;
+      this.vectorLength = vectorLength;
       this.makeVariableWeightsVariable = makeVariableWeightsVariable;
       this.addLinearScalingTerms = addLinearScalingTerms;
@@ -91 +98 @@
         var value = ((ConstantTreeNode)node).Value;
         //initialConstants.Add(value);
-        var var = tf.Variable(value);
+        var var = tf.Variable(value, name: $"c_{variables.Count}", dtype: tf.float64);
         variables.Add(var);
         return var;
       }
 
-      if (node.Symbol is Variable || node.Symbol is BinaryFactorVariable) {
+      if (node.Symbol is Variable/* || node.Symbol is BinaryFactorVariable*/) {
         var varNode = node as VariableTreeNodeBase;
-        var factorVarNode = node as BinaryFactorVariableTreeNode;
+        //var factorVarNode = node as BinaryFactorVariableTreeNode;
         // factor variable values are only 0 or 1 and set in x accordingly
-        var varValue = factorVarNode != null ? factorVarNode.VariableValue : string.Empty;
-        var par = FindOrCreateParameter(parameters, varNode.VariableName, varValue);
+        //var varValue = factorVarNode != null ? factorVarNode.VariableValue : string.Empty;
+        //var par = FindOrCreateParameter(parameters, varNode.VariableName, varValue);
+        var shape = vectorLength.HasValue
+          ? new TensorShape(numRows, vectorLength.Value)
+          : new TensorShape(numRows);
+        var par = tf.placeholder(tf.float64, shape: shape, name: varNode.VariableName);
+        parameters.Add(par, varNode.VariableName);
 
         if (makeVariableWeightsVariable) {
           //initialConstants.Add(varNode.Weight);
-          var w = tf.Variable(varNode.Weight);
+          var w = tf.Variable(varNode.Weight, name: $"w_{varNode.VariableName}_{variables.Count}", dtype: tf.float64);
           variables.Add(w);
           return w * par;
@@ -117 +129 @@
         var products = new List<Tensor>();
         foreach (var variableValue in factorVarNode.Symbol.GetVariableValues(factorVarNode.VariableName)) {
-          var par = FindOrCreateParameter(parameters, factorVarNode.VariableName, variableValue);
+          //var par = FindOrCreateParameter(parameters, factorVarNode.VariableName, variableValue);
+          var par = tf.placeholder(tf.float64, shape: new TensorShape(numRows), name: factorVarNode.VariableName);
+          parameters.Add(par, factorVarNode.VariableName);
 
           var value = factorVarNode.GetValue(variableValue);
           //initialConstants.Add(value);
-          var wVar = tf.Variable(value);
+          var wVar = tf.Variable(value, name: $"f_{factorVarNode.VariableName}_{variables.Count}");
           variables.Add(wVar);
 
@@ -127 +141 @@
         }
 
-        return tf.add_n(products.ToArray());
+        return products.Aggregate((a, b) => a + b);
       }
 
@@ -136 +150 @@
         }
 
-        return tf.add_n(terms.ToArray());
+        return terms.Aggregate((a, b) => a + b);
       }
 
@@ -148 +162 @@
 
         if (terms.Count == 1) return -terms[0];
-        else return tf.add_n(terms.ToArray());
+        else return terms.Aggregate((a, b) => a + b);
       }
 
@@ -233 +247 @@
       if (node.Symbol is Mean) {
         return tf.reduce_mean(
-          ConvertNode(node.GetSubtree(0)));
+          ConvertNode(node.GetSubtree(0)),
+          axis: new[] { 1 });
       }
 
       //if (node.Symbol is StandardDeviation) {
       //  return tf.reduce_std(
-      //    ConvertNode(node.GetSubtree(0)));
+      //    ConvertNode(node.GetSubtree(0)),
+      //    axis: new [] { 1 }
+      // );
       //}
 
       if (node.Symbol is Sum) {
         return tf.reduce_sum(
-          ConvertNode(node.GetSubtree(0)));
+          ConvertNode(node.GetSubtree(0)),
+          axis: new[] { 1 });
       }
 
@@ -249 +267 @@
         if (addLinearScalingTerms) {
           // scaling variables α, β are given at the beginning of the parameter vector
-          var alpha = tf.Variable(1.0);
-          var beta = tf.Variable(0.0);
+          var alpha = tf.Variable(1.0, name: $"alpha_{1.0}", dtype: tf.float64);
+          var beta = tf.Variable(0.0, name: $"beta_{0.0}", dtype: tf.float64);
+          variables.Add(alpha);
           variables.Add(beta);
-          variables.Add(alpha);
           var t = ConvertNode(node.GetSubtree(0));
           return t * alpha + beta;
@@ -268 +286 @@
       if (!parameters.TryGetValue(data, out var par)) {
         // not found -> create new parameter and entries in names and values lists
-        par = tf.placeholder(tf.float64, name: varName);
+        par = tf.placeholder(tf.float64, shape: new TensorShape(-1), name: varName);
         parameters.Add(data, par);
       }
@@ -279 +297 @@
         where
           !(n.Symbol is Variable) &&
-          !(n.Symbol is BinaryFactorVariable) &&
-          !(n.Symbol is FactorVariable) &&
+          //!(n.Symbol is BinaryFactorVariable) &&
+          //!(n.Symbol is FactorVariable) &&
           !(n.Symbol is Constant) &&
           !(n.Symbol is Addition) &&