Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
03/22/22 13:28:56 (2 years ago)
Author:
pfleck
Message:

#3040 Updated to newer TensorFlow.NET version.

  • Removed IL Merge from TensorFlow.NET.
  • Temporarily removed DiffSharp.
  • Changed to a locally built Attic with a specific Protobuf version that is compatible with TensorFlow.NET. (Also adapted other versions of nuget dependencies.)
Location:
branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToDiffSharpConverter.cs

    r17930 r18239  
    1919 */
    2020#endregion
     21
     22#if INCLUDE_DIFFSHARP
    2123
    2224using System;
    … …  
    505507  }
    506508}
     509
     510#endif

  • branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs

    r18238 r18239  
    2020#endregion
    2121
    22 #define EXPLICIT_SHAPE
    23 
    2422using System;
    2523using System.Collections.Generic;
    2624using System.Linq;
    2725using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
    28 using NumSharp;
    2926using Tensorflow;
     27using Tensorflow.NumPy;
    3028using static Tensorflow.Binding;
     29using DoubleVector = MathNet.Numerics.LinearAlgebra.Vector<double>;
    3130
    3231namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
    3332  public class TreeToTensorConverter {
    3433
     34    //private static readonly TF_DataType DataType = tf.float64;
    3535    private static readonly TF_DataType DataType = tf.float32;
    3636
    37     public static bool TryConvert(ISymbolicExpressionTree tree, int numRows, Dictionary<string, int> variableLengths,
    38       bool makeVariableWeightsVariable, bool addLinearScalingTerms,
    39       out Tensor graph, out Dictionary<Tensor, string> parameters, out List<Tensor> variables) {
     37    public static bool TryPrepareTree(
     38      ISymbolicExpressionTree tree,
     39      IRegressionProblemData problemData, List<int> rows,
     40      bool updateVariableWeights, bool applyLinearScaling,
     41      bool eagerEvaluation,
     42      out Dictionary<string, Tensor> inputFeatures, out Tensor target,
     43      out Dictionary<ISymbolicExpressionTreeNode, ResourceVariable[]> variables) {
    4044
    4145      try {
    42         var converter = new TreeToTensorConverter(numRows, variableLengths, makeVariableWeightsVariable, addLinearScalingTerms);
    43         graph = converter.ConvertNode(tree.Root.GetSubtree(0));
    44 
    45         parameters = converter.parameters;
     46        var converter = new TreeToTensorConverter(
     47          problemData, rows,
     48          updateVariableWeights, applyLinearScaling,
     49          eagerEvaluation
     50        );
     51
     52        if (eagerEvaluation)
     53          tf.enable_eager_execution();
     54        else
     55          tf.compat.v1.disable_eager_execution();
     56
     57        converter.PrepareNode(tree.Root.GetSubtree(0));
     58
     59        inputFeatures = converter.inputFeatures;
     60        target = InputFeatureToTensor(problemData.TargetVariable, problemData, rows);
    4661        variables = converter.variables;
     62
    4763        return true;
    4864      } catch (NotSupportedException) {
    49         graph = null;
    50         parameters = null;
     65        inputFeatures = null;
     66        target= null;
    5167        variables = null;
    5268        return false;
    5369      }
    54     }
    55 
    56     private readonly int numRows;
    57     private readonly Dictionary<string, int> variableLengths;
     70
     71    }
     72
     73    public static bool TryEvaluate(
     74      ISymbolicExpressionTree tree,
     75      Dictionary<string, Tensor> inputFeatures, Dictionary<ISymbolicExpressionTreeNode, ResourceVariable[]> variables,
     76      bool makeVariableWeightsVariable, bool addLinearScalingTerms,
     77      bool eagerEvaluation,
     78      out Tensor prediction) {
     79
     80      try {
     81        var converter = new TreeToTensorConverter(
     82          inputFeatures, variables,
     83          makeVariableWeightsVariable, addLinearScalingTerms,
     84          eagerEvaluation
     85        );
     86
     87        if (eagerEvaluation)
     88          tf.enable_eager_execution();
     89        else
     90          tf.compat.v1.disable_eager_execution();
     91
     92        prediction = converter.EvaluateNode(tree.Root.GetSubtree(0));
     93
     94        return true;
     95      } catch (NotSupportedException) {
     96        prediction = null;
     97        return false;
     98      }
     99
     100    }
     101
     102    //public static bool TryConvert/*Lazy/Graph*/(
     103    //  ISymbolicExpressionTree tree,
     104    //  Dictionary<string, Tensor> inputFeatures, Dictionary<ISymbolicExpressionTreeNode, ResourceVariable[]> variables,
     105    //  bool makeVariableWeightsVariable, bool addLinearScalingTerms,
     106    //  out Tensor prediction) {
     107
     108    //  try {
     109    //    var converter = new TreeToTensorConverter(
     110    //      inputFeatures, variables,
     111    //      makeVariableWeightsVariable, addLinearScalingTerms,
     112    //      eagerEvaluation: false
     113    //    );
     114       
     115    //    tf.compat.v1.disable_eager_execution();
     116    //    prediction = converter.EvaluateNode(tree.Root.GetSubtree(0));
     117    //    return true;
     118    //  } catch (NotSupportedException) {
     119    //    prediction = null;
     120    //    return false;
     121    //  }
     122    //}
     123
     124
     125    private readonly IDataAnalysisProblemData problemData;
     126    private readonly List<int> rows;
     127
     128    private readonly Dictionary<string, Tensor> inputFeatures = new Dictionary<string, Tensor>();
     129    private readonly Dictionary<ISymbolicExpressionTreeNode, ResourceVariable[]> variables = new Dictionary<ISymbolicExpressionTreeNode, ResourceVariable[]>();
     130
    58131    private readonly bool makeVariableWeightsVariable;
    59132    private readonly bool addLinearScalingTerms;
    60 
    61     private readonly Dictionary<Tensor, string> parameters = new Dictionary<Tensor, string>();
    62     private readonly List<Tensor> variables = new List<Tensor>();
    63 
    64     private TreeToTensorConverter(int numRows, Dictionary<string, int> variableLengths, bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
    65       this.numRows = numRows;
    66       this.variableLengths = variableLengths;
     133    private readonly bool eagerEvaluation;
     134
     135    private TreeToTensorConverter(
     136      IDataAnalysisProblemData problemData, List<int> rows,
     137      bool makeVariableWeightsVariable, bool addLinearScalingTerms,
     138      bool eagerEvaluation 
     139    ) {
     140      this.problemData = problemData;
     141      this.rows = rows;
     142
    67143      this.makeVariableWeightsVariable = makeVariableWeightsVariable;
    68144      this.addLinearScalingTerms = addLinearScalingTerms;
    69     }
    70 
    71 
    72     private Tensor ConvertNode(ISymbolicExpressionTreeNode node) {
     145      this.eagerEvaluation = eagerEvaluation;
     146    }
     147
     148    private TreeToTensorConverter(
     149      Dictionary<string, Tensor> inputFeatures, Dictionary<ISymbolicExpressionTreeNode, ResourceVariable[]> variables,
     150      bool makeVariableWeightsVariable, bool addLinearScalingTerms,
     151      bool eagerEvaluation
     152    ) {
     153      this.inputFeatures = inputFeatures;
     154      this.variables = variables;
     155
     156      this.makeVariableWeightsVariable = makeVariableWeightsVariable;
     157      this.addLinearScalingTerms = addLinearScalingTerms;
     158      this.eagerEvaluation = eagerEvaluation;
     159    }
     160   
     161    private static Tensor InputFeatureToTensor(string var, IDataAnalysisProblemData problemData, List<int> rows) {
     162      if (problemData.Dataset.VariableHasType<double>(var)) {
     163        var data = problemData.Dataset.GetDoubleValues(var, rows).Select(x => (float)x).ToArray();
     164        return tf.convert_to_tensor(np.array(data).reshape(new Shape(rows.Count, 1)), DataType);
     165      } else if (problemData.Dataset.VariableHasType<DoubleVector>(var)) {
     166        var data = problemData.Dataset.GetDoubleVectorValues(var, rows).SelectMany(x => x.Select(y => (float)y)).ToArray();
     167        return tf.convert_to_tensor(np.array(data).reshape(new Shape(rows.Count, -1)), DataType);
     168      } else throw new NotSupportedException($"Type of the variable is not supported: {var}");
     169    }
     170    private static Tensor InputFeatureToPlaceholder(string var, IDataAnalysisProblemData problemData, List<int> rows) {
     171      if (problemData.Dataset.VariableHasType<double>(var)) {
     172        return tf.placeholder(DataType, new Shape(rows.Count, 1), name: var);
     173      } else if (problemData.Dataset.VariableHasType<DoubleVector>(var)) {
     174        //var vectorLength = problemData.Dataset.GetDoubleVectorValues(var, rows).Select(v => v.Count).Distinct().Single();
     175        var vectorLength = problemData.Dataset.GetDoubleVectorValue(var, rows[0]).Count;
     176        return tf.placeholder(DataType, new Shape(rows.Count, vectorLength), name: var);
     177      } else throw new NotSupportedException($"Type of the variable is not supported: {var}");
     178    }
     179
     180    private void PrepareNode(ISymbolicExpressionTreeNode node) {
     181      if (node.Symbol is Constant ) {
     182        var constantNode = (ConstantTreeNode)node;
     183        var value = (float)constantNode.Value;
     184        var value_arr = np.array(value).reshape(new Shape(1, 1));
     185        var c = tf.Variable(value_arr, name: $"c_{variables.Count}", dtype: DataType);
     186        variables.Add(node, new[] { c });
     187      } else if (node.Symbol is Variable) {
     188        var varNode = (VariableTreeNodeBase)node;
     189        if (makeVariableWeightsVariable) {
     190          var w_arr = np.array((float)varNode.Weight).reshape(new Shape(1, 1));
     191          var w = tf.Variable(w_arr, name: $"w_{varNode.VariableName}", dtype: DataType);
     192          variables.Add(node, new[] { w });
     193        }
     194        if (!inputFeatures.ContainsKey(varNode.VariableName)) {
     195          inputFeatures.Add(
     196            varNode.VariableName,
     197            eagerEvaluation
     198              ? InputFeatureToTensor(varNode.VariableName, problemData, rows)
     199              : InputFeatureToPlaceholder(varNode.VariableName, problemData, rows));
     200        }
     201      } else if (node.Symbol is StartSymbol) {
     202        if (addLinearScalingTerms) {
     203          var alpha_arr = np.array(1.0f).reshape(new Shape(1, 1));
     204          var alpha = tf.Variable(alpha_arr, name: "alpha", dtype: DataType);
     205          var beta_arr = np.array(0.0f).reshape(new Shape(1, 1));
     206          var beta = tf.Variable(beta_arr, name: "beta", dtype: DataType);
     207          variables.Add(node, new[] { beta, alpha });
     208        }
     209      }
     210
     211      foreach (var subTree in node.Subtrees) {
     212        PrepareNode(subTree);
     213      }
     214    }
     215
     216
     217    private Tensor EvaluateNode(ISymbolicExpressionTreeNode node) {
    73218      if (node.Symbol is Constant) {
    74         var value = (float)((ConstantTreeNode)node).Value;
    75         var value_arr = np.array(value).reshape(1, 1);
    76         var var = tf.Variable(value_arr, name: $"c_{variables.Count}", dtype: DataType);
    77         variables.Add(var);
    78         return var;
     219        return variables[node][0];
    79220      }
    80221
    81222      if (node.Symbol is Variable/* || node.Symbol is BinaryFactorVariable*/) {
    82223        var varNode = node as VariableTreeNodeBase;
    83         //var factorVarNode = node as BinaryFactorVariableTreeNode;
    84         // factor variable values are only 0 or 1 and set in x accordingly
    85         //var varValue = factorVarNode != null ? factorVarNode.VariableValue : string.Empty;
    86         //var par = FindOrCreateParameter(parameters, varNode.VariableName, varValue);
    87         var par = tf.placeholder(DataType, new TensorShape(numRows, variableLengths[varNode.VariableName]), name: varNode.VariableName);
    88         parameters.Add(par, varNode.VariableName);
    89 
     224       
     225        var par = inputFeatures[varNode.VariableName]; // eager or placeholder
    90226        if (makeVariableWeightsVariable) {
    91           var w_arr = np.array((float)varNode.Weight).reshape(1, 1);
    92           var w = tf.Variable(w_arr, name: $"w_{varNode.VariableName}", dtype: DataType);
    93           variables.Add(w);
     227          var w = variables[node][0];
    94228          return w * par;
    95229        } else {
    … …  
    119253
    120254      if (node.Symbol is Addition) {
    121         var terms = node.Subtrees.Select(ConvertNode).ToList();
     255        var terms = node.Subtrees.Select(EvaluateNode).ToList();
    122256        if (terms.Count == 1) return terms[0];
    123257        return terms.Aggregate((a, b) => a + b);
    … …  
    125259
    126260      if (node.Symbol is Subtraction) {
    127         var terms = node.Subtrees.Select(ConvertNode).ToList();
     261        var terms = node.Subtrees.Select(EvaluateNode).ToList();
    128262        if (terms.Count == 1) return -terms[0];
    129263        return terms.Aggregate((a, b) => a - b);
    … …  
    131265
    132266      if (node.Symbol is Multiplication) {
    133         var terms = node.Subtrees.Select(ConvertNode).ToList();
     267        var terms = node.Subtrees.Select(EvaluateNode).ToList();
    134268        if (terms.Count == 1) return terms[0];
    135269        return terms.Aggregate((a, b) => a * b);
    … …  
    137271
    138272      if (node.Symbol is Division) {
    139         var terms = node.Subtrees.Select(ConvertNode).ToList();
    140         if (terms.Count == 1) return 1.0f / terms[0];
     273        var terms = node.Subtrees.Select(EvaluateNode).ToList();
     274        //if (terms.Count == 1) return 1.0f / terms[0];
     275        if (terms.Count == 1) return 1.0 / terms[0];
    141276        return terms.Aggregate((a, b) => a / b);
    142277      }
    143278
    144279      if (node.Symbol is Absolute) {
    145         var x1 = ConvertNode(node.GetSubtree(0));
     280        var x1 = EvaluateNode(node.GetSubtree(0));
    146281        return tf.abs(x1);
    147282      }
    148283
    149284      if (node.Symbol is AnalyticQuotient) {
    150         var x1 = ConvertNode(node.GetSubtree(0));
    151         var x2 = ConvertNode(node.GetSubtree(1));
     285        var x1 = EvaluateNode(node.GetSubtree(0));
     286        var x2 = EvaluateNode(node.GetSubtree(1));
    152287        return x1 / tf.pow(1.0f + x2 * x2, 0.5f);
     288        //return x1 / tf.pow(1.0 + x2 * x2, 0.5);
    153289      }
    154290
    155291      if (node.Symbol is Logarithm) {
    156292        return tf.log(
    157           ConvertNode(node.GetSubtree(0)));
     293          EvaluateNode(node.GetSubtree(0)));
    158294      }
    159295
    … …  
    161297        return tf.pow(
    162298          (float)Math.E,
    163           ConvertNode(node.GetSubtree(0)));
     299          //Math.E,
     300          EvaluateNode(node.GetSubtree(0)));
    164301      }
    165302
    166303      if (node.Symbol is Square) {
    167304        return tf.square(
    168           ConvertNode(node.GetSubtree(0)));
     305          EvaluateNode(node.GetSubtree(0)));
    169306      }
    170307
    171308      if (node.Symbol is SquareRoot) {
    172309        return tf.sqrt(
    173           ConvertNode(node.GetSubtree(0)));
     310          EvaluateNode(node.GetSubtree(0)));
    174311      }
    175312
    176313      if (node.Symbol is Cube) {
    177314        return tf.pow(
    178           ConvertNode(node.GetSubtree(0)), 3.0f);
     315          EvaluateNode(node.GetSubtree(0)), 3.0f);
     316        //ConvertNode(node.GetSubtree(0)), 3.0);
    179317      }
    180318
    181319      if (node.Symbol is CubeRoot) {
    182320        return tf.pow(
    183           ConvertNode(node.GetSubtree(0)), 1.0f / 3.0f);
     321          EvaluateNode(node.GetSubtree(0)), 1.0f / 3.0f);
     322        //ConvertNode(node.GetSubtree(0)), 1.0 / 3.0);
    184323        // TODO
    185324        // f: x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3),
    … …  
    189328      if (node.Symbol is Sine) {
    190329        return tf.sin(
    191           ConvertNode(node.GetSubtree(0)));
     330          EvaluateNode(node.GetSubtree(0)));
    192331      }
    193332
    194333      if (node.Symbol is Cosine) {
    195334        return tf.cos(
    196           ConvertNode(node.GetSubtree(0)));
     335          EvaluateNode(node.GetSubtree(0)));
    197336      }
    198337
    199338      if (node.Symbol is Tangent) {
    200339        return tf.tan(
    201           ConvertNode(node.GetSubtree(0)));
     340          EvaluateNode(node.GetSubtree(0)));
    202341      }
    203342
    204343      if (node.Symbol is Mean) {
    205344        return tf.reduce_mean(
    206           ConvertNode(node.GetSubtree(0)),
     345          EvaluateNode(node.GetSubtree(0)),
    207346          axis: new[] { 1 },
    208347          keepdims: true);
    … …  
    210349
    211350      if (node.Symbol is StandardDeviation) {
    212         return reduce_std(
    213           ConvertNode(node.GetSubtree(0)),
     351        return tf.reduce_std(
     352          EvaluateNode(node.GetSubtree(0)),
    214353          axis: new[] { 1 },
    215354          keepdims: true
    216        );
     355        );
    217356      }
    218357
    219358      if (node.Symbol is Variance) {
    220         return reduce_var(
    221           ConvertNode(node.GetSubtree(0)),
     359        return tf.reduce_variance(
     360          EvaluateNode(node.GetSubtree(0)),
    222361          axis: new[] { 1 } ,
    223362          keepdims: true
    … …  
    227366      if (node.Symbol is Sum) {
    228367        return tf.reduce_sum(
    229           ConvertNode(node.GetSubtree(0)),
     368          EvaluateNode(node.GetSubtree(0)),
    230369          axis: new[] { 1 },
    231370          keepdims: true);
    … …  
    233372
    234373      if (node.Symbol is SubVector) {
    235         var tensor = ConvertNode(node.GetSubtree(0));
    236         int rows = tensor.shape[0], vectorLength = tensor.shape[1];
     374        var tensor = EvaluateNode(node.GetSubtree(0));
     375        int rows = (int)tensor.shape[0], vectorLength = (int)tensor.shape[1];
    237376        var windowedNode = (IWindowedSymbolTreeNode)node;
    238377        int startIdx = SymbolicDataAnalysisExpressionTreeVectorInterpreter.ToVectorIdx(windowedNode.Offset, vectorLength);
    … …  
    244383          segments.Add(tensor[new Slice(), new Slice(start, start + count)]);
    245384        }
    246         return tf.concat(segments.ToArray(), axis: 1);
     385        return tf.concat(segments, axis: 1);
    247386      }
    248387
    249388
    250389      if (node.Symbol is StartSymbol) {
    251         Tensor prediction;
     390        Tensor prediction = EvaluateNode(node.GetSubtree(0));
     391
     392        if (prediction.rank != 2 && prediction.shape[1] != 1)
     393          throw new InvalidOperationException("Prediction must be a rank 1 (single value per row).");
     394
     395        prediction = tf.reshape(prediction, new Shape(-1));
     396
    252397        if (addLinearScalingTerms) {
    253           // scaling variables α, β are given at the beginning of the parameter vector
    254           var alpha_arr = np.array(1.0f).reshape(1, 1);
    255           var alpha = tf.Variable(alpha_arr, name: "alpha", dtype: DataType);
    256           var beta_arr = np.array(0.0f).reshape(1, 1);
    257           var beta = tf.Variable(beta_arr, name: "beta", dtype: DataType);
    258           variables.Add(beta);
    259           variables.Add(alpha);
    260           var t = ConvertNode(node.GetSubtree(0));
    261           prediction = t * alpha + beta;
     398          var vars = variables[node];
     399          Tensor alpha = vars[1], beta = vars[0];
     400          return prediction * alpha + beta;
    262401        } else {
    263           prediction = ConvertNode(node.GetSubtree(0));
    264         }
    265 
    266         return tf.reshape(prediction, shape: new[] { -1 });
     402          return prediction;
     403        }
    267404      }
    268405
    269406      throw new NotSupportedException($"Node symbol {node.Symbol} is not supported.");
    270     }
    271 
    272     private static Tensor reduce_var(Tensor input_tensor,  int[] axis = null, bool keepdims = false) {
    273       var means = tf.reduce_mean(input_tensor, axis, true);
    274       var squared_deviation = tf.square(input_tensor - means);
    275       return tf.reduce_mean(squared_deviation, axis, keepdims);
    276     }
    277     private static Tensor reduce_std(Tensor input_tensor, int[] axis = null, bool keepdims = false) {
    278       return tf.sqrt(reduce_var(input_tensor, axis, keepdims));
    279407    }
    280408

  • branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj

    r18229 r18239  
    113113      <Private>False</Private>
    114114    </Reference>
    115     <Reference Include="DiffSharp.Merged, Version=0.8.4.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=AMD64">
    116       <SpecificVersion>False</SpecificVersion>
    117       <HintPath>..\..\bin\DiffSharp.Merged.dll</HintPath>
    118     </Reference>
    119115    <Reference Include="HEAL.Attic, Version=1.0.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
    120116      <SpecificVersion>False</SpecificVersion>
    … …  
    144140    <Reference Include="System.Data" />
    145141    <Reference Include="System.Xml" />
    146     <Reference Include="TensorFlow.NET.Merged, Version=0.15.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
     142    <Reference Include="Tensorflow.Binding, Version=0.70.1.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=AMD64">
    147143      <SpecificVersion>False</SpecificVersion>
    148       <HintPath>..\..\bin\TensorFlow.NET.Merged.dll</HintPath>
     144      <HintPath>..\..\bin\Tensorflow.Binding.dll</HintPath>
    149145    </Reference>
    150146  </ItemGroup>

  • branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Plugin.cs.frame

    r18229 r18239  
    5656  [PluginDependency("HeuristicLab.MathNet.Numerics", "4.9.0")]
    5757  [PluginDependency("HeuristicLab.TensorFlowNet", "0.15.0")]
    58   [PluginDependency("HeuristicLab.DiffSharp", "0.7.7")]
     58  //[PluginDependency("HeuristicLab.DiffSharp", "0.7.7")]
    5959  public class HeuristicLabProblemsDataAnalysisSymbolicPlugin : PluginBase {
    6060  }
Note: See TracChangeset for help on using the changeset viewer.