Changeset 17476


Ignore:
Timestamp:
03/12/20 17:51:39 (2 weeks ago)
Author:
pfleck
Message:

#3040 Worked on TF-based constant optimization.

Location:
branches/3040_VectorBasedGP
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs

    r17475 r17476  
    2323using System.Collections;
    2424using System.Collections.Generic;
     25using System.Diagnostics;
    2526using System.Linq;
    2627using System.Threading;
     
    3132using HeuristicLab.Parameters;
    3233using HEAL.Attic;
     34using NumSharp;
    3335using Tensorflow;
    3436using static Tensorflow.Binding;
     
    9294      CancellationToken cancellationToken = default(CancellationToken), EvaluationsCounter counter = null) {
    9395
    94       bool success = TreeToTensorConverter.TryConvert(tree, updateVariableWeights, applyLinearScaling,
    95         out Tensor prediction, out Dictionary<TreeToTensorConverter.DataForVariable, Tensor> variables/*, out double[] initialConstants*/);
     96      var vectorVariables = tree.IterateNodesBreadth()
     97        .OfType<VariableTreeNodeBase>()
     98        .Where(node => problemData.Dataset.VariableHasType<DoubleVector>(node.VariableName))
     99        .Select(node => node.VariableName);
     100
     101      int? vectorLength = null;
     102      if (vectorVariables.Any()) {
     103        vectorLength = vectorVariables.Select(var => problemData.Dataset.GetDoubleVectorValues(var, rows)).First().First().Count;
     104      }
     105      int numRows = rows.Count();
     106
     107      bool success = TreeToTensorConverter.TryConvert(tree,
     108        numRows, vectorLength,
     109        updateVariableWeights, applyLinearScaling,
     110        out Tensor prediction,
     111        out Dictionary<Tensor, string> parameters, out List<Tensor> variables/*, out double[] initialConstants*/);
    96112
    97113      var target = tf.placeholder(tf.float64, name: problemData.TargetVariable);
     
    99115      // mse
    100116      var costs = tf.reduce_sum(tf.square(prediction - target)) / (2.0 * samples);
    101       var optimizer = tf.train.GradientDescentOptimizer((float)learningRate);
     117      var optimizer = tf.train.GradientDescentOptimizer((float)learningRate).minimize(costs);
    102118
    103119      // features as feed items
    104120      var variablesFeed = new Hashtable();
    105       foreach (var kvp in variables) {
    106         var variableName = kvp.Key.variableName;
    107         var variable = kvp.Value;
    108         if (problemData.Dataset.VariableHasType<double>(variableName))
    109           variablesFeed.Add(variable, problemData.Dataset.GetDoubleValues(variableName, rows));
    110         if (problemData.Dataset.VariableHasType<string>(variableName))
    111           variablesFeed.Add(variable, problemData.Dataset.GetStringValues(variableName, rows));
    112         if (problemData.Dataset.VariableHasType<DoubleVector>(variableName))
    113           variablesFeed.Add(variable, problemData.Dataset.GetDoubleVectorValues(variableName, rows));
    114         throw new NotSupportedException($"Type of the variable is not supported: {variableName}");
     121      foreach (var kvp in parameters) {
     122        var variable = kvp.Key;
     123        var variableName = kvp.Value;
     124        if (problemData.Dataset.VariableHasType<double>(variableName)) {
     125          var data = problemData.Dataset.GetDoubleValues(variableName, rows).ToArray();
     126          if (vectorLength.HasValue) {
     127            var vectorData = new double[numRows][];
     128            for (int i = 0; i < numRows; i++)
     129              vectorData[i] = Enumerable.Repeat(data[i], vectorLength.Value).ToArray();
     130            variablesFeed.Add(variable, np.array(vectorData));
     131          } else
     132            variablesFeed.Add(variable, np.array(data, copy: false));
     133          //} else if (problemData.Dataset.VariableHasType<string>(variableName)) {
     134          //  variablesFeed.Add(variable, problemData.Dataset.GetStringValues(variableName, rows));
     135        } else if (problemData.Dataset.VariableHasType<DoubleVector>(variableName)) {
     136          var data = problemData.Dataset.GetDoubleVectorValues(variableName, rows).Select(x => x.ToArray()).ToArray();
     137          variablesFeed.Add(variable, np.array(data));
     138        } else
     139          throw new NotSupportedException($"Type of the variable is not supported: {variableName}");
    115140      }
    116       variablesFeed.Add(target, problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows));
     141      var targetData = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, rows).ToArray();
     142      variablesFeed.Add(target, np.array(targetData, copy: false));
    117143
    118144
    119145      using (var session = tf.Session()) {
     146        session.run(tf.global_variables_initializer());
     147
     148        Trace.WriteLine("Weights:");
     149        foreach (var v in variables)
     150          Trace.WriteLine($"{v.name}: {session.run(v).ToString(true)}");
     151
    120152        for (int i = 0; i < maxIterations; i++) {
    121           optimizer.minimize(costs);
    122           var result = session.run(optimizer, variablesFeed);
     153
     154          //optimizer.minimize(costs);
     155          session.run(optimizer, variablesFeed);
     156
     157          Trace.WriteLine("Weights:");
     158          foreach (var v in variables)
     159            Trace.WriteLine($"{v.name}: {session.run(v).ToString(true)}");
    123160        }
    124161      }
    125       optimizer.minimize(costs);
    126162
    127163      if (!success)
  • branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs

    r17474 r17476  
    2323using System.Collections.Generic;
    2424using System.Linq;
    25 using System.Runtime.Serialization;
    26 using AutoDiff;
    2725using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    2826using Tensorflow;
    2927using static Tensorflow.Binding;
     28using DoubleVector = MathNet.Numerics.LinearAlgebra.Vector<double>;
    3029
    3130namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
     
    5554    #endregion
    5655
    57     public static bool TryConvert(ISymbolicExpressionTree tree, bool makeVariableWeightsVariable, bool addLinearScalingTerms,
    58       out Tensor graph, out Dictionary<DataForVariable, Tensor> variables/*, out double[] initialConstants*/) {
     56    public static bool TryConvert(ISymbolicExpressionTree tree, int numRows, int? vectorLength,
     57      bool makeVariableWeightsVariable, bool addLinearScalingTerms,
     58      out Tensor graph, out Dictionary<Tensor, string> parameters, out List<Tensor> variables
     59/*, out double[] initialConstants*/) {
    5960
    6061      try {
    61         var converter = new TreeToTensorConverter(makeVariableWeightsVariable, addLinearScalingTerms);
     62        var converter = new TreeToTensorConverter(numRows, vectorLength, makeVariableWeightsVariable, addLinearScalingTerms);
    6263        graph = converter.ConvertNode(tree.Root.GetSubtree(0));
    6364
    6465        //var parametersEntries = converter.parameters.ToList(); // guarantee same order for keys and values
    65         variables = converter.parameters; // parametersEntries.Select(kvp => kvp.Value).ToList();
     66        parameters = converter.parameters; // parametersEntries.Select(kvp => kvp.Value).ToList();
     67        variables = converter.variables;
    6668        //initialConstants = converter.initialConstants.ToArray();
    6769        return true;
    6870      } catch (NotSupportedException) {
    6971        graph = null;
     72        parameters = null;
    7073        variables = null;
    7174        //initialConstants = null;
     
    7477    }
    7578
     79    private readonly int numRows;
     80    private readonly int? vectorLength;
    7681    private readonly bool makeVariableWeightsVariable;
    7782    private readonly bool addLinearScalingTerms;
    7883
    7984    //private readonly List<double> initialConstants = new List<double>();
    80     private readonly Dictionary<DataForVariable, Tensor> parameters = new Dictionary<DataForVariable, Tensor>();
     85    private readonly Dictionary<Tensor, string> parameters = new Dictionary<Tensor, string>();
    8186    private readonly List<Tensor> variables = new List<Tensor>();
    8287
    83     private TreeToTensorConverter(bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
     88    private TreeToTensorConverter(int numRows, int? vectorLength, bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
     89      this.numRows = numRows;
     90      this.vectorLength = vectorLength;
    8491      this.makeVariableWeightsVariable = makeVariableWeightsVariable;
    8592      this.addLinearScalingTerms = addLinearScalingTerms;
     
    9198        var value = ((ConstantTreeNode)node).Value;
    9299        //initialConstants.Add(value);
    93         var var = tf.Variable(value);
     100        var var = tf.Variable(value, name: $"c_{variables.Count}", dtype: tf.float64);
    94101        variables.Add(var);
    95102        return var;
    96103      }
    97104
    98       if (node.Symbol is Variable || node.Symbol is BinaryFactorVariable) {
     105      if (node.Symbol is Variable/* || node.Symbol is BinaryFactorVariable*/) {
    99106        var varNode = node as VariableTreeNodeBase;
    100         var factorVarNode = node as BinaryFactorVariableTreeNode;
     107        //var factorVarNode = node as BinaryFactorVariableTreeNode;
    101108        // factor variable values are only 0 or 1 and set in x accordingly
    102         var varValue = factorVarNode != null ? factorVarNode.VariableValue : string.Empty;
    103         var par = FindOrCreateParameter(parameters, varNode.VariableName, varValue);
     109        //var varValue = factorVarNode != null ? factorVarNode.VariableValue : string.Empty;
     110        //var par = FindOrCreateParameter(parameters, varNode.VariableName, varValue);
     111        var shape = vectorLength.HasValue
     112          ? new TensorShape(numRows, vectorLength.Value)
     113          : new TensorShape(numRows);
     114        var par = tf.placeholder(tf.float64, shape: shape, name: varNode.VariableName);
     115        parameters.Add(par, varNode.VariableName);
    104116
    105117        if (makeVariableWeightsVariable) {
    106118          //initialConstants.Add(varNode.Weight);
    107           var w = tf.Variable(varNode.Weight);
     119          var w = tf.Variable(varNode.Weight, name: $"w_{varNode.VariableName}_{variables.Count}", dtype: tf.float64);
    108120          variables.Add(w);
    109121          return w * par;
     
    117129        var products = new List<Tensor>();
    118130        foreach (var variableValue in factorVarNode.Symbol.GetVariableValues(factorVarNode.VariableName)) {
    119           var par = FindOrCreateParameter(parameters, factorVarNode.VariableName, variableValue);
     131          //var par = FindOrCreateParameter(parameters, factorVarNode.VariableName, variableValue);
     132          var par = tf.placeholder(tf.float64, shape: new TensorShape(numRows), name: factorVarNode.VariableName);
     133          parameters.Add(par, factorVarNode.VariableName);
    120134
    121135          var value = factorVarNode.GetValue(variableValue);
    122136          //initialConstants.Add(value);
    123           var wVar = tf.Variable(value);
     137          var wVar = tf.Variable(value, name: $"f_{factorVarNode.VariableName}_{variables.Count}");
    124138          variables.Add(wVar);
    125139
     
    127141        }
    128142
    129         return tf.add_n(products.ToArray());
     143        return products.Aggregate((a, b) => a + b);
    130144      }
    131145
     
    136150        }
    137151
    138         return tf.add_n(terms.ToArray());
     152        return terms.Aggregate((a, b) => a + b);
    139153      }
    140154
     
    148162
    149163        if (terms.Count == 1) return -terms[0];
    150         else return tf.add_n(terms.ToArray());
     164        else return terms.Aggregate((a, b) => a + b);
    151165      }
    152166
     
    233247      if (node.Symbol is Mean) {
    234248        return tf.reduce_mean(
    235           ConvertNode(node.GetSubtree(0)));
     249          ConvertNode(node.GetSubtree(0)),
     250          axis: new[] { 1 });
    236251      }
    237252
    238253      //if (node.Symbol is StandardDeviation) {
    239254      //  return tf.reduce_std(
    240       //    ConvertNode(node.GetSubtree(0)));
     255      //    ConvertNode(node.GetSubtree(0)),
     256      //    axis: new [] { 1 }
     257      // );
    241258      //}
    242259
    243260      if (node.Symbol is Sum) {
    244261        return tf.reduce_sum(
    245           ConvertNode(node.GetSubtree(0)));
     262          ConvertNode(node.GetSubtree(0)),
     263          axis: new[] { 1 });
    246264      }
    247265
     
    249267        if (addLinearScalingTerms) {
    250268          // scaling variables α, β are given at the beginning of the parameter vector
    251           var alpha = tf.Variable(1.0);
    252           var beta = tf.Variable(0.0);
     269          var alpha = tf.Variable(1.0, name: $"alpha_{1.0}", dtype: tf.float64);
     270          var beta = tf.Variable(0.0, name: $"beta_{0.0}", dtype: tf.float64);
     271          variables.Add(alpha);
    253272          variables.Add(beta);
    254           variables.Add(alpha);
    255273          var t = ConvertNode(node.GetSubtree(0));
    256274          return t * alpha + beta;
     
    268286      if (!parameters.TryGetValue(data, out var par)) {
    269287        // not found -> create new parameter and entries in names and values lists
    270         par = tf.placeholder(tf.float64, name: varName);
     288        par = tf.placeholder(tf.float64, shape: new TensorShape(-1), name: varName);
    271289        parameters.Add(data, par);
    272290      }
     
    279297        where
    280298          !(n.Symbol is Variable) &&
    281           !(n.Symbol is BinaryFactorVariable) &&
    282           !(n.Symbol is FactorVariable) &&
     299          //!(n.Symbol is BinaryFactorVariable) &&
     300          //!(n.Symbol is FactorVariable) &&
    283301          !(n.Symbol is Constant) &&
    284302          !(n.Symbol is Addition) &&
Note: See TracChangeset for help on using the changeset viewer.