Changeset 17474

Timestamp:

03/11/20 15:02:34 (5 years ago)

Author:

pfleck

Message:

#3040 Started working on the TF constant opt evaluator.

Location:

branches/3040_VectorBasedGP

Files:

• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj (modified) (3 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/SingleObjective/Evaluators/TensorFlowConstantOptimizationEvaluator.cs (added)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs (copied) (copied from branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToAutoDiffTermConverter.cs) (6 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj (modified) (2 diffs)

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.csproj

@@ -104 +104 @@
       <Private>False</Private>
     </Reference>
+    <Reference Include="MathNet.Numerics">
+      <HintPath>..\..\bin\MathNet.Numerics.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core">
@@ -117 +121 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
+    <Reference Include="TensorFlow.NET.Signed">
+      <HintPath>..\..\bin\TensorFlow.NET.Signed.dll</HintPath>
+      <Private>False</Private>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
@@ -128 +136 @@
     <Compile Include="SingleObjective\Evaluators\NonlinearLeastSquaresConstantOptimizationEvaluator.cs" />
     <Compile Include="SingleObjective\Evaluators\SymbolicRegressionMeanRelativeErrorEvaluator.cs" />
+    <Compile Include="SingleObjective\Evaluators\TensorFlowConstantOptimizationEvaluator.cs" />
     <Compile Include="SingleObjective\SymbolicRegressionSolutionsAnalyzer.cs" />
     <Compile Include="SymbolicRegressionPhenotypicDiversityAnalyzer.cs" />

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToTensorConverter.cs

@@ -26 +26 @@
 using AutoDiff;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using Tensorflow;
+using static Tensorflow.Binding;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
-  public class TreeToAutoDiffTermConverter {
-    public delegate double ParametricFunction(double[] vars, double[] @params);
-
-    public delegate Tuple<double[], double> ParametricFunctionGradient(double[] vars, double[] @params);
+  public class TreeToTensorConverter {
 
     #region helper class
@@ -37 +36 @@
       public readonly string variableName;
       public readonly string variableValue; // for factor vars
-      public readonly int lag;
-
-      public DataForVariable(string varName, string varValue, int lag) {
+
+      public DataForVariable(string varName, string varValue) {
         this.variableName = varName;
         this.variableValue = varValue;
-        this.lag = lag;
       }
 
@@ -49 +46 @@
         if (other == null) return false;
         return other.variableName.Equals(this.variableName) &&
-               other.variableValue.Equals(this.variableValue) &&
-               other.lag == this.lag;
+               other.variableValue.Equals(this.variableValue);
       }
 
       public override int GetHashCode() {
-        return variableName.GetHashCode() ^ variableValue.GetHashCode() ^ lag;
+        return variableName.GetHashCode() ^ variableValue.GetHashCode();
       }
     }
     #endregion
 
-    #region derivations of functions
-    // create function factory for arctangent
-    private static readonly Func<Term, UnaryFunc> arctan = UnaryFunc.Factory(
-      eval: Math.Atan,
-      diff: x => 1 / (1 + x * x));
-
-    private static readonly Func<Term, UnaryFunc> sin = UnaryFunc.Factory(
-      eval: Math.Sin,
-      diff: Math.Cos);
-
-    private static readonly Func<Term, UnaryFunc> cos = UnaryFunc.Factory(
-      eval: Math.Cos,
-      diff: x => -Math.Sin(x));
-
-    private static readonly Func<Term, UnaryFunc> tan = UnaryFunc.Factory(
-      eval: Math.Tan,
-      diff: x => 1 + Math.Tan(x) * Math.Tan(x));
-    private static readonly Func<Term, UnaryFunc> tanh = UnaryFunc.Factory(
-      eval: Math.Tanh,
-      diff: x => 1 - Math.Tanh(x) * Math.Tanh(x));
-    private static readonly Func<Term, UnaryFunc> erf = UnaryFunc.Factory(
-      eval: alglib.errorfunction,
-      diff: x => 2.0 * Math.Exp(-(x * x)) / Math.Sqrt(Math.PI));
-
-    private static readonly Func<Term, UnaryFunc> norm = UnaryFunc.Factory(
-      eval: alglib.normaldistribution,
-      diff: x => -(Math.Exp(-(x * x)) * Math.Sqrt(Math.Exp(x * x)) * x) / Math.Sqrt(2 * Math.PI));
-
-    private static readonly Func<Term, UnaryFunc> abs = UnaryFunc.Factory(
-      eval: Math.Abs,
-      diff: x => Math.Sign(x)
-      );
-
-    private static readonly Func<Term, UnaryFunc> cbrt = UnaryFunc.Factory(
-      eval: x => x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3),
-      diff: x => { var cbrt_x = x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3); return 1.0 / (3 * cbrt_x * cbrt_x); }
-      );
-
-
-
-    #endregion
-
-    public static bool TryConvertToAutoDiff(ISymbolicExpressionTree tree, bool makeVariableWeightsVariable, bool addLinearScalingTerms,
-      out List<DataForVariable> parameters, out double[] initialConstants,
-      out ParametricFunction func,
-      out ParametricFunctionGradient func_grad) {
-
-      // use a transformator object which holds the state (variable list, parameter list, ...) for recursive transformation of the tree
-      var transformator = new TreeToAutoDiffTermConverter(makeVariableWeightsVariable, addLinearScalingTerms);
-      AutoDiff.Term term;
+    public static bool TryConvert(ISymbolicExpressionTree tree, bool makeVariableWeightsVariable, bool addLinearScalingTerms,
+      out Tensor graph, out Dictionary<DataForVariable, Tensor> variables/*, out double[] initialConstants*/) {
+
       try {
-        term = transformator.ConvertToAutoDiff(tree.Root.GetSubtree(0));
-        var parameterEntries = transformator.parameters.ToArray(); // guarantee same order for keys and values
-        var compiledTerm = term.Compile(transformator.variables.ToArray(),
-          parameterEntries.Select(kvp => kvp.Value).ToArray());
-        parameters = new List<DataForVariable>(parameterEntries.Select(kvp => kvp.Key));
-        initialConstants = transformator.initialConstants.ToArray();
-        func = (vars, @params) => compiledTerm.Evaluate(vars, @params);
-        func_grad = (vars, @params) => compiledTerm.Differentiate(vars, @params);
+        var converter = new TreeToTensorConverter(makeVariableWeightsVariable, addLinearScalingTerms);
+        graph = converter.ConvertNode(tree.Root.GetSubtree(0));
+
+        //var parametersEntries = converter.parameters.ToList(); // guarantee same order for keys and values
+        variables = converter.parameters; // parametersEntries.Select(kvp => kvp.Value).ToList();
+        //initialConstants = converter.initialConstants.ToArray();
         return true;
-      } catch (ConversionException) {
-        func = null;
-        func_grad = null;
-        parameters = null;
-        initialConstants = null;
-      }
-      return false;
-    }
-
-    // state for recursive transformation of trees 
-    private readonly
-    List<double> initialConstants;
-    private readonly Dictionary<DataForVariable, AutoDiff.Variable> parameters;
-    private readonly List<AutoDiff.Variable> variables;
+      } catch (NotSupportedException) {
+        graph = null;
+        variables = null;
+        //initialConstants = null;
+        return false;
+      }
+    }
+
     private readonly bool makeVariableWeightsVariable;
     private readonly bool addLinearScalingTerms;
 
-    private TreeToAutoDiffTermConverter(bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
+    //private readonly List<double> initialConstants = new List<double>();
+    private readonly Dictionary<DataForVariable, Tensor> parameters = new Dictionary<DataForVariable, Tensor>();
+    private readonly List<Tensor> variables = new List<Tensor>();
+
+    private TreeToTensorConverter(bool makeVariableWeightsVariable, bool addLinearScalingTerms) {
       this.makeVariableWeightsVariable = makeVariableWeightsVariable;
       this.addLinearScalingTerms = addLinearScalingTerms;
-      this.initialConstants = new List<double>();
-      this.parameters = new Dictionary<DataForVariable, AutoDiff.Variable>();
-      this.variables = new List<AutoDiff.Variable>();
-    }
-
-    private AutoDiff.Term ConvertToAutoDiff(ISymbolicExpressionTreeNode node) {
+    }
+
+
+    private Tensor ConvertNode(ISymbolicExpressionTreeNode node) {
       if (node.Symbol is Constant) {
-        initialConstants.Add(((ConstantTreeNode)node).Value);
-        var var = new AutoDiff.Variable();
+        var value = ((ConstantTreeNode)node).Value;
+        //initialConstants.Add(value);
+        var var = tf.Variable(value);
         variables.Add(var);
         return var;
       }
+
       if (node.Symbol is Variable || node.Symbol is BinaryFactorVariable) {
         var varNode = node as VariableTreeNodeBase;
@@ -159 +104 @@
 
         if (makeVariableWeightsVariable) {
-          initialConstants.Add(varNode.Weight);
-          var w = new AutoDiff.Variable();
+          //initialConstants.Add(varNode.Weight);
+          var w = tf.Variable(varNode.Weight);
           variables.Add(w);
-          return AutoDiff.TermBuilder.Product(w, par);
+          return w * par;
         } else {
           return varNode.Weight * par;
         }
       }
+
       if (node.Symbol is FactorVariable) {
         var factorVarNode = node as FactorVariableTreeNode;
-        var products = new List<Term>();
+        var products = new List<Tensor>();
         foreach (var variableValue in factorVarNode.Symbol.GetVariableValues(factorVarNode.VariableName)) {
           var par = FindOrCreateParameter(parameters, factorVarNode.VariableName, variableValue);
 
-          initialConstants.Add(factorVarNode.GetValue(variableValue));
-          var wVar = new AutoDiff.Variable();
+          var value = factorVarNode.GetValue(variableValue);
+          //initialConstants.Add(value);
+          var wVar = tf.Variable(value);
           variables.Add(wVar);
 
-          products.Add(AutoDiff.TermBuilder.Product(wVar, par));
-        }
-        return AutoDiff.TermBuilder.Sum(products);
-      }
-      if (node.Symbol is LaggedVariable) {
-        var varNode = node as LaggedVariableTreeNode;
-        var par = FindOrCreateParameter(parameters, varNode.VariableName, string.Empty, varNode.Lag);
-
-        if (makeVariableWeightsVariable) {
-          initialConstants.Add(varNode.Weight);
-          var w = new AutoDiff.Variable();
-          variables.Add(w);
-          return AutoDiff.TermBuilder.Product(w, par);
-        } else {
-          return varNode.Weight * par;
-        }
-      }
+          products.add(wVar * par);
+        }
+
+        return tf.add_n(products.ToArray());
+      }
+
       if (node.Symbol is Addition) {
-        List<AutoDiff.Term> terms = new List<Term>();
+        var terms = new List<Tensor>();
         foreach (var subTree in node.Subtrees) {
-          terms.Add(ConvertToAutoDiff(subTree));
-        }
-        return AutoDiff.TermBuilder.Sum(terms);
-      }
+          terms.Add(ConvertNode(subTree));
+        }
+
+        return tf.add_n(terms.ToArray());
+      }
+
       if (node.Symbol is Subtraction) {
-        List<AutoDiff.Term> terms = new List<Term>();
+        var terms = new List<Tensor>();
         for (int i = 0; i < node.SubtreeCount; i++) {
-          AutoDiff.Term t = ConvertToAutoDiff(node.GetSubtree(i));
+          var t = ConvertNode(node.GetSubtree(i));
           if (i > 0) t = -t;
           terms.Add(t);
         }
+
         if (terms.Count == 1) return -terms[0];
-        else return AutoDiff.TermBuilder.Sum(terms);
-      }
+        else return tf.add_n(terms.ToArray());
+      }
+
       if (node.Symbol is Multiplication) {
-        List<AutoDiff.Term> terms = new List<Term>();
+        var terms = new List<Tensor>();
         foreach (var subTree in node.Subtrees) {
-          terms.Add(ConvertToAutoDiff(subTree));
-        }
+          terms.Add(ConvertNode(subTree));
+        }
+
         if (terms.Count == 1) return terms[0];
-        else return terms.Aggregate((a, b) => new AutoDiff.Product(a, b));
-      }
+        else return terms.Aggregate((a, b) => a * b);
+      }
+
       if (node.Symbol is Division) {
-        List<AutoDiff.Term> terms = new List<Term>();
+        var terms = new List<Tensor>();
         foreach (var subTree in node.Subtrees) {
-          terms.Add(ConvertToAutoDiff(subTree));
-        }
+          terms.Add(ConvertNode(subTree));
+        }
+
         if (terms.Count == 1) return 1.0 / terms[0];
-        else return terms.Aggregate((a, b) => new AutoDiff.Product(a, 1.0 / b));
-      }
+        else return terms.Aggregate((a, b) => a * (1.0 / b));
+      }
+
       if (node.Symbol is Absolute) {
-        var x1 = ConvertToAutoDiff(node.GetSubtree(0));
-        return abs(x1);
-      }
+        var x1 = ConvertNode(node.GetSubtree(0));
+        return tf.abs(x1);
+      }
+
       if (node.Symbol is AnalyticQuotient) {
-        var x1 = ConvertToAutoDiff(node.GetSubtree(0));
-        var x2 = ConvertToAutoDiff(node.GetSubtree(1));
-        return x1 / (TermBuilder.Power(1 + x2 * x2, 0.5));
-      }
+        var x1 = ConvertNode(node.GetSubtree(0));
+        var x2 = ConvertNode(node.GetSubtree(1));
+        return x1 / tf.pow(1 + x2 * x2, 0.5);
+      }
+
       if (node.Symbol is Logarithm) {
-        return AutoDiff.TermBuilder.Log(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
+        return math_ops.log(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
       if (node.Symbol is Exponential) {
-        return AutoDiff.TermBuilder.Exp(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
+        return math_ops.pow(
+          Math.E,
+          ConvertNode(node.GetSubtree(0)));
+      }
+
       if (node.Symbol is Square) {
-        return AutoDiff.TermBuilder.Power(
-          ConvertToAutoDiff(node.GetSubtree(0)), 2.0);
-      }
+        return tf.square(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
       if (node.Symbol is SquareRoot) {
-        return AutoDiff.TermBuilder.Power(
-          ConvertToAutoDiff(node.GetSubtree(0)), 0.5);
-      }
+        return math_ops.sqrt(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
       if (node.Symbol is Cube) {
-        return AutoDiff.TermBuilder.Power(
-          ConvertToAutoDiff(node.GetSubtree(0)), 3.0);
-      }
+        return math_ops.pow(
+          ConvertNode(node.GetSubtree(0)), 3.0);
+      }
+
       if (node.Symbol is CubeRoot) {
-        return cbrt(ConvertToAutoDiff(node.GetSubtree(0)));
-      }
+        return math_ops.pow(
+          ConvertNode(node.GetSubtree(0)), 1.0 / 3.0);
+        // TODO
+        // f: x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3),
+        // g:  { var cbrt_x = x < 0 ? -Math.Pow(-x, 1.0 / 3) : Math.Pow(x, 1.0 / 3); return 1.0 / (3 * cbrt_x * cbrt_x); }
+      }
+
       if (node.Symbol is Sine) {
-        return sin(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
+        return tf.sin(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
       if (node.Symbol is Cosine) {
-        return cos(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
+        return tf.cos(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
       if (node.Symbol is Tangent) {
-        return tan(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
-      if (node.Symbol is HyperbolicTangent) {
-        return tanh(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
-      if (node.Symbol is Erf) {
-        return erf(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
-      if (node.Symbol is Norm) {
-        return norm(
-          ConvertToAutoDiff(node.GetSubtree(0)));
-      }
+        return tf.tan(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      if (node.Symbol is Mean) {
+        return tf.reduce_mean(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
+      //if (node.Symbol is StandardDeviation) {
+      //  return tf.reduce_std(
+      //    ConvertNode(node.GetSubtree(0)));
+      //}
+
+      if (node.Symbol is Sum) {
+        return tf.reduce_sum(
+          ConvertNode(node.GetSubtree(0)));
+      }
+
       if (node.Symbol is StartSymbol) {
         if (addLinearScalingTerms) {
           // scaling variables α, β are given at the beginning of the parameter vector
-          var alpha = new AutoDiff.Variable();
-          var beta = new AutoDiff.Variable();
+          var alpha = tf.Variable(1.0);
+          var beta = tf.Variable(0.0);
           variables.Add(beta);
           variables.Add(alpha);
-          var t = ConvertToAutoDiff(node.GetSubtree(0));
+          var t = ConvertNode(node.GetSubtree(0));
           return t * alpha + beta;
-        } else return ConvertToAutoDiff(node.GetSubtree(0));
-      }
-      throw new ConversionException();
-    }
-
+        } else return ConvertNode(node.GetSubtree(0));
+      }
+
+      throw new NotSupportedException($"Node symbol {node.Symbol} is not supported.");
+    }
 
     // for each factor variable value we need a parameter which represents a binary indicator for that variable & value combination
     // each binary indicator is only necessary once. So we only create a parameter if this combination is not yet available
-    private static Term FindOrCreateParameter(Dictionary<DataForVariable, AutoDiff.Variable> parameters,
-      string varName, string varValue = "", int lag = 0) {
-      var data = new DataForVariable(varName, varValue, lag);
-
-      AutoDiff.Variable par = null;
-      if (!parameters.TryGetValue(data, out par)) {
+    private static Tensor FindOrCreateParameter(Dictionary<DataForVariable, Tensor> parameters, string varName, string varValue = "") {
+      var data = new DataForVariable(varName, varValue);
+
+      if (!parameters.TryGetValue(data, out var par)) {
         // not found -> create new parameter and entries in names and values lists
-        par = new AutoDiff.Variable();
+        par = tf.placeholder(tf.float64, name: varName);
         parameters.Add(data, par);
       }
@@ -320 +281 @@
           !(n.Symbol is BinaryFactorVariable) &&
           !(n.Symbol is FactorVariable) &&
-          !(n.Symbol is LaggedVariable) &&
           !(n.Symbol is Constant) &&
           !(n.Symbol is Addition) &&
@@ -340 +300 @@
           !(n.Symbol is AnalyticQuotient) &&
           !(n.Symbol is Cube) &&
-          !(n.Symbol is CubeRoot)
+          !(n.Symbol is CubeRoot) &&
+          !(n.Symbol is Mean) &&
+          //!(n.Symbol is StandardDeviation) &&
+          !(n.Symbol is Sum)
         select n).Any();
       return !containsUnknownSymbol;
     }
-    #region exception class
-    [Serializable]
-    public class ConversionException : Exception {
-
-      public ConversionException() {
-      }
-
-      public ConversionException(string message) : base(message) {
-      }
-
-      public ConversionException(string message, Exception inner) : base(message, inner) {
-      }
-
-      protected ConversionException(
-        SerializationInfo info,
-        StreamingContext context) : base(info, context) {
-      }
-    }
-    #endregion
   }
 }

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj

@@ -133 +133 @@
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
+    <Reference Include="TensorFlow.NET.Signed, Version=0.15.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\bin\TensorFlow.NET.Signed.dll</HintPath>
+    </Reference>
   </ItemGroup>
   <ItemGroup>
@@ -153 +157 @@
     <Compile Include="Converters\TreeSimplifier.cs" />
     <Compile Include="Converters\DerivativeCalculator.cs" />
+    <Compile Include="Converters\TreeToTensorConverter.cs" />
     <Compile Include="Converters\TreeToAutoDiffTermConverter.cs" />
     <Compile Include="Creators\SymbolicDataAnalysisExpressionBalancedTreeCreator.cs" />