Changeset 17193 for branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing

Timestamp:

08/07/19 13:32:09 (5 years ago)

Author:

mkommend

Message:

#2974: Merged trunk changes into branch.

Location:

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic

Files:

• . (modified) (1 prop)
• 3.4/Hashing/HashExtensions.cs (modified) (8 diffs)
• 3.4/Hashing/HashUtil.cs (modified) (1 diff)
• 3.4/Hashing/SymbolicExpressionTreeHash.cs (modified) (9 diffs)

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/HashExtensions.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -21 +21 @@
 
 using System;
-using System.Collections.Generic;
+using System.Linq;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
   public static class SymbolicExpressionHashExtensions {
+    /// <summary>
+    /// Holds data that is necessary to handle tree nodes in hashing / simplification.
+    /// </summary>
+    /// <typeparam name="T">The tree node type</typeparam>
     public sealed class HashNode<T> : IComparable<HashNode<T>>, IEquatable<HashNode<T>> where T : class {
       public T Data;
@@ -38 +42 @@
       public SimplifyAction Simplify;
 
-      public IComparer<T> Comparer;
-
       public bool IsLeaf => Arity == 0;
 
-      public HashNode(IComparer<T> comparer) {
-        Comparer = comparer;
-      }
-
-      private HashNode() { }
-
       public int CompareTo(HashNode<T> other) {
-        var res = Comparer.Compare(Data, other.Data);
+        var res = HashValue.CompareTo(other.HashValue);
         return res == 0 ? CalculatedHashValue.CompareTo(other.CalculatedHashValue) : res;
       }
@@ -103 +99 @@
 
     public static HashNode<T>[] Simplify<T>(this HashNode<T>[] nodes, Func<byte[], ulong> hashFunction) where T : class {
-      var reduced = nodes.UpdateNodeSizes().Reduce().Sort(hashFunction);
-
-      for (int i = 0; i < reduced.Length; ++i) {
-        var node = reduced[i];
-        if (node.IsLeaf) {
-          continue;
-        }
-        node.Simplify?.Invoke(ref reduced, i);
-      }
-      // detect if anything was simplified
-      var count = 0;
-      foreach (var node in reduced) {
-        if (!node.Enabled) { ++count; }
-      }
-      if (count == 0) {
-        return reduced;
-      }
-
-      var simplified = new HashNode<T>[reduced.Length - count];
-      int j = 0;
-      foreach (var node in reduced) {
-        if (node.Enabled) {
-          simplified[j++] = node;
-        }
-      }
-      return simplified.UpdateNodeSizes().Reduce().Sort(hashFunction);
+      bool simplified = false;
+      nodes = nodes.UpdateNodeSizes().Reduce().Sort(hashFunction);
+      do {
+        if (simplified) {
+          simplified = false;
+          nodes = nodes.Where(x => x.Enabled).ToArray().UpdateNodeSizes().Reduce().Sort(hashFunction);
+        }
+
+        for (int i = 0; i < nodes.Length; ++i) {
+          var node = nodes[i];
+          if (node.IsLeaf) {
+            continue;
+          }
+          node.Simplify?.Invoke(ref nodes, i);
+          for (int j = i - node.Size; j <= i; ++j) {
+            // detect if anything was simplified
+            if (!nodes[j].Enabled) {
+              simplified = true;
+              break;
+            }
+          }
+        }
+      } while (simplified);
+      return nodes.UpdateNodeSizes().Sort(hashFunction);
     }
 
@@ -173 +167 @@
 
     /// <summary>
-    /// Get a function node's child indicest
+    /// Get a function node's child indices
     /// </summary>
     /// <typeparam name="T">The data type encapsulated by a hash node</typeparam>
-    /// <param name="nodes">An array of hash nodes with up-to-date node sizes</param>
+    /// <param name="nodes">An array of hash nodes with up-to-date node sizes (see UpdateNodeSizes)</param>
     /// <param name="i">The index in the array of hash nodes of the node whose children we want to iterate</param>
     /// <returns>An array containing child indices</returns>
@@ -191 +185 @@
     }
 
+    /// <summary>
+    /// Determines size of each branch and sets the results for each node.
+    /// </summary>
+    /// <typeparam name="T">The data type encapsulated by a hash node</typeparam>
+    /// <param name="nodes">An array of hash nodes in postfix order.</param>
+    /// <returns>The array with updated node sizes. The array is not copied.</returns>
     public static HashNode<T>[] UpdateNodeSizes<T>(this HashNode<T>[] nodes) where T : class {
       for (int i = 0; i < nodes.Length; ++i) {
@@ -199 +199 @@
         }
         node.Size = node.Arity;
-
+        // visit all children and sum up their size (assumes postfix order).
         for (int j = i - 1, k = 0; k < node.Arity; j -= 1 + nodes[j].Size, ++k) {
           node.Size += nodes[j].Size;
@@ -207 +207 @@
     }
 
-    private static HashNode<T>[] Reduce<T>(this HashNode<T>[] nodes) where T : class {
+    // disables duplicate branches and removes the disabled nodes
+    public static HashNode<T>[] Reduce<T>(this HashNode<T>[] nodes) where T : class {
       int count = 0;
       for (int i = 0; i < nodes.Length; ++i) {

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/HashUtil.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.

branches/2974_Constants_Optimization/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/SymbolicExpressionTreeHash.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -38 +38 @@
     private static readonly Constant constant = new Constant();
 
-    private static readonly ISymbolicExpressionTreeNodeComparer comparer = new SymbolicExpressionTreeNodeComparer();
     private static ISymbolicExpressionTreeNode ActualRoot(this ISymbolicExpressionTree tree) => tree.Root.GetSubtree(0).GetSubtree(0);
+    public static ulong HashFunction(byte[] input) => HashUtil.DJBHash(input);
 
     #region tree hashing
@@ -47 +47 @@
 
     public static ulong[] Hash(this ISymbolicExpressionTreeNode node, bool simplify = false, bool strict = false) {
-      ulong hashFunction(byte[] input) => HashUtil.DJBHash(input);
-
-      var hashNodes = simplify ? node.MakeNodes(strict).Simplify(hashFunction) : node.MakeNodes(strict).Sort(hashFunction);
+      var hashNodes = simplify ? node.MakeNodes(strict).Simplify(HashFunction) : node.MakeNodes(strict).Sort(HashFunction);
       var hashes = new ulong[hashNodes.Length];
       for (int i = 0; i < hashes.Length; ++i) {
@@ -74 +72 @@
       }
       var hash = (ulong)name.GetHashCode();
-      var hashNode = new HashNode<ISymbolicExpressionTreeNode>(comparer) {
+      var hashNode = new HashNode<ISymbolicExpressionTreeNode> {
         Data = node,
         Arity = node.SubtreeCount,
@@ -168 +166 @@
       for (int i = 0; i < trees.Count; ++i) {
         var nodes = trees[i].MakeNodes(strict);
-        hashes[i] = (simplify ? nodes.Simplify(HashUtil.DJBHash) : nodes.Sort(HashUtil.DJBHash)).Select(x => x.CalculatedHashValue).ToArray();
+        hashes[i] = (simplify ? nodes.Simplify(HashFunction) : nodes.Sort(HashFunction)).Select(x => x.CalculatedHashValue).ToArray();
         Array.Sort(hashes[i]);
       }
@@ -180 +178 @@
     }
 
+    public static double[,] ComputeSimilarityMatrix(IList<ulong[]> hashes) {
+      // compute similarity matrix
+      var n = hashes.Count;
+      var sim = new double[n, n];
+      for (int i = 0; i < n - 1; ++i) {
+        for (int j = i + 1; j < n; ++j) {
+          sim[i, j] = sim[j, i] = ComputeSimilarity(hashes[i], hashes[j]);
+        }
+      }
+      return sim;
+    }
+
     public static double[,] ComputeSimilarityMatrix(IList<ISymbolicExpressionTree> trees, bool simplify = false, bool strict = false) {
-      var sim = new double[trees.Count, trees.Count];
       var hashes = new ulong[trees.Count][];
       // build hash arrays
       for (int i = 0; i < trees.Count; ++i) {
         var nodes = trees[i].MakeNodes(strict);
-        hashes[i] = (simplify ? nodes.Simplify(HashUtil.DJBHash) : nodes.Sort(HashUtil.DJBHash)).Select(x => x.CalculatedHashValue).ToArray();
+        hashes[i] = (simplify ? nodes.Simplify(HashFunction) : nodes.Sort(HashFunction)).Select(x => x.CalculatedHashValue).ToArray();
         Array.Sort(hashes[i]);
       }
-      // compute similarity matrix
-      for (int i = 0; i < trees.Count - 1; ++i) {
-        for (int j = i + 1; j < trees.Count; ++j) {
-          sim[i, j] = sim[j, i] = ComputeSimilarity(hashes[i], hashes[j]);
-        }
-      }
-      return sim;
+      return ComputeSimilarityMatrix(hashes);
     }
     #endregion
@@ -245 +248 @@
     // (in other words simplification should be applied in a bottom-up fashion)
     public static ISymbolicExpressionTree Simplify(ISymbolicExpressionTree tree) {
-      ulong hashFunction(byte[] bytes) => HashUtil.JSHash(bytes);
-      var root = tree.Root.GetSubtree(0).GetSubtree(0);
-      var nodes = root.MakeNodes();
-      var simplified = nodes.Simplify(hashFunction);
-      return simplified.ToTree();
+      return tree.MakeNodes().Simplify(HashFunction).ToTree();
     }
 
@@ -286 +285 @@
 
         var symbol = child.Data.Symbol;
-        if (symbol is Constant) {
+        if (child.Data is ConstantTreeNode firstConst) {
+          // fold sibling constant nodes into the first constant
           for (int k = j + 1; k < children.Length; ++k) {
-            var d = children[k];
-            if (nodes[d].Data.Symbol is Constant) {
-              nodes[d].Enabled = false;
+            var sibling = nodes[children[k]];
+            if (sibling.Data is ConstantTreeNode otherConst) {
+              sibling.Enabled = false;
               node.Arity--;
+              firstConst.Value *= otherConst.Value;
+            } else {
+              break;
+            }
+          }
+        } else if (child.Data is VariableTreeNode variable) {
+          // fold sibling constant nodes into the variable weight
+          for (int k = j + 1; k < children.Length; ++k) {
+            var sibling = nodes[children[k]];
+            if (sibling.Data is ConstantTreeNode constantNode) {
+              sibling.Enabled = false;
+              node.Arity--;
+              variable.Weight *= constantNode.Value;
             } else {
               break;
@@ -297 +310 @@
           }
         } else if (symbol is Division) {
-          var div = nodes[c];
-          var denominator =
-            div.Arity == 1 ?
-            nodes[c - 1] :                    // 1 / x is expressed as div(x) (with a single child)
-            nodes[c - nodes[c - 1].Size - 2]; // assume division always has arity 1 or 2
-
-          foreach (var d in children) {
-            if (nodes[d].Enabled && nodes[d] == denominator) {
-              nodes[c].Enabled = nodes[d].Enabled = denominator.Enabled = false;
+          // 1/x is expressed as div(x) (with a single child)
+          // we assume division always has arity 1 or 2
+          var d = child.Arity == 1 ? c - 1 : c - nodes[c - 1].Size - 2;
+          var denominator = nodes[d];
+
+          // iterate children of node i to see if any of them matches the denominator of div node c
+          for (int k = 0; k < children.Length; ++k) {
+            var sibling = nodes[children[k]];
+            if (sibling.Enabled && sibling == denominator) {
+              nodes.SetEnabled(children[j], false); // disable the div subtree
+              nodes.SetEnabled(children[k], false); // disable the sibling matching the denominator
+
               node.Arity -= 2; // matching child + division node
               break;