Changeset 16654 for branches/2866_SymRegHyperbolicFunctions/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing

Timestamp:

03/07/19 12:29:27 (6 years ago)

Author:

gkronber

Message:

#2866: merged r16364:16653 from trunk to branch to prepare for trunk reintegration (resolving conflicts in the project file)

Location:

branches/2866_SymRegHyperbolicFunctions/HeuristicLab.Problems.DataAnalysis.Symbolic

Files:

• . (modified) (1 prop)
• 3.4/Hashing/HashExtensions.cs (modified) (1 diff)
• 3.4/Hashing/HashUtil.cs (modified) (1 diff)
• 3.4/Hashing/SymbolicExpressionTreeHash.cs (modified) (7 diffs)

branches/2866_SymRegHyperbolicFunctions/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/HashExtensions.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.

branches/2866_SymRegHyperbolicFunctions/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/HashUtil.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.

branches/2866_SymRegHyperbolicFunctions/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Hashing/SymbolicExpressionTreeHash.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -21 +21 @@
 
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
@@ -38 +39 @@
 
     private static readonly ISymbolicExpressionTreeNodeComparer comparer = new SymbolicExpressionTreeNodeComparer();
-
-    public static ulong ComputeHash(this ISymbolicExpressionTree tree) {
-      return ComputeHash(tree.Root.GetSubtree(0).GetSubtree(0));
-    }
-
-    public static double ComputeSimilarity(ISymbolicExpressionTree t1, ISymbolicExpressionTree t2, bool simplify = false) {
-      return ComputeSimilarity(t1.Root.GetSubtree(0).GetSubtree(0), t2.Root.GetSubtree(0).GetSubtree(0), simplify);
-    }
-
-    public static double ComputeSimilarity(ISymbolicExpressionTreeNode t1, ISymbolicExpressionTreeNode t2, bool simplify = false) {
-      HashNode<ISymbolicExpressionTreeNode>[] lhs;
-      HashNode<ISymbolicExpressionTreeNode>[] rhs;
-
+    private static ISymbolicExpressionTreeNode ActualRoot(this ISymbolicExpressionTree tree) => tree.Root.GetSubtree(0).GetSubtree(0);
+
+    #region tree hashing
+    public static ulong[] Hash(this ISymbolicExpressionTree tree, bool simplify = false, bool strict = false) {
+      return tree.ActualRoot().Hash(simplify, strict);
+    }
+
+    public static ulong[] Hash(this ISymbolicExpressionTreeNode node, bool simplify = false, bool strict = false) {
       ulong hashFunction(byte[] input) => HashUtil.DJBHash(input);
 
-      if (simplify) {
-        lhs = t1.MakeNodes().Simplify(hashFunction);
-        rhs = t2.MakeNodes().Simplify(hashFunction);
-      } else {
-        lhs = t1.MakeNodes().Sort(hashFunction); // sort calculates hash values
-        rhs = t2.MakeNodes().Sort(hashFunction);
-      }
-
-      var lh = lhs.Select(x => x.CalculatedHashValue).ToArray();
-      var rh = rhs.Select(x => x.CalculatedHashValue).ToArray();
-
-      Array.Sort(lh);
-      Array.Sort(rh);
-
-      return ComputeSimilarity(lh, rh);
-    }
-
-    // this will only work if lh and rh are sorted 
-    private static double ComputeSimilarity(ulong[] lh, ulong[] rh) {
-      double count = 0;
-      for (int i = 0, j = 0; i < lh.Length && j < rh.Length;) {
-        var h1 = lh[i];
-        var h2 = rh[j];
-        if (h1 == h2) {
-          ++count;
-          ++i;
-          ++j;
-        } else if (h1 < h2) {
-          ++i;
-        } else if (h1 > h2) {
-          ++j;
-        }
-      }
-
-      return 2d * count / (lh.Length + rh.Length);
-    }
-
-    public static double ComputeAverageSimilarity(ISymbolicExpressionTree[] trees, bool simplify = false, bool strict = false) {
-      var total = (double)trees.Length * (trees.Length - 1) / 2;
-      double avg = 0;
-      var hashes = new ulong[trees.Length][];
-      // build hash arrays
-      for (int i = 0; i < trees.Length; ++i) {
-        var nodes = trees[i].MakeNodes(strict);
-        hashes[i] = (simplify ? nodes.Simplify(HashUtil.DJBHash) : nodes.Sort(HashUtil.DJBHash)).Select(x => x.CalculatedHashValue).ToArray();
-        Array.Sort(hashes[i]);
-      }
-      // compute similarity matrix
-      for (int i = 0; i < trees.Length - 1; ++i) {
-        for (int j = i + 1; j < trees.Length; ++j) {
-          avg += ComputeSimilarity(hashes[i], hashes[j]);
-        }
-      }
-      return avg / total;
-    }
-
-    public static double[,] ComputeSimilarityMatrix(ISymbolicExpressionTree[] trees, bool simplify = false, bool strict = false) {
-      var sim = new double[trees.Length, trees.Length];
-      var hashes = new ulong[trees.Length][];
-      // build hash arrays
-      for (int i = 0; i < trees.Length; ++i) {
-        var nodes = trees[i].MakeNodes(strict);
-        hashes[i] = (simplify ? nodes.Simplify(HashUtil.DJBHash) : nodes.Sort(HashUtil.DJBHash)).Select(x => x.CalculatedHashValue).ToArray();
-        Array.Sort(hashes[i]);
-      }
-      // compute similarity matrix
-      for (int i = 0; i < trees.Length - 1; ++i) {
-        for (int j = i + 1; j < trees.Length; ++j) {
-          sim[i, j] = sim[j, i] = ComputeSimilarity(hashes[i], hashes[j]);
-        }
-      }
-      return sim;
-    }
-
-    public static ulong ComputeHash(this ISymbolicExpressionTreeNode treeNode, bool strict = false) {
-      ulong hashFunction(byte[] input) => HashUtil.JSHash(input);
-      var hashNodes = treeNode.MakeNodes(strict);
-      var simplified = hashNodes.Simplify(hashFunction);
-      return simplified.Last().CalculatedHashValue;
+      var hashNodes = simplify ? node.MakeNodes(strict).Simplify(hashFunction) : node.MakeNodes(strict).Sort(hashFunction);
+      var hashes = new ulong[hashNodes.Length];
+      for (int i = 0; i < hashes.Length; ++i) {
+        hashes[i] = hashNodes[i].CalculatedHashValue;
+      }
+      return hashes;
+    }
+
+    public static ulong ComputeHash(this ISymbolicExpressionTree tree, bool simplify = false, bool strict = false) {
+      return ComputeHash(tree.ActualRoot(), simplify, strict);
+    }
+
+    public static ulong ComputeHash(this ISymbolicExpressionTreeNode treeNode, bool simplify = false, bool strict = false) {
+      return treeNode.Hash(simplify, strict).Last();
     }
 
@@ -167 +98 @@
 
     public static HashNode<ISymbolicExpressionTreeNode>[] MakeNodes(this ISymbolicExpressionTree tree, bool strict = false) {
-      return MakeNodes(tree.Root.GetSubtree(0).GetSubtree(0), strict);
+      return MakeNodes(tree.ActualRoot(), strict);
     }
 
@@ -173 +104 @@
       return node.IterateNodesPostfix().Select(x => x.ToHashNode(strict)).ToArray().UpdateNodeSizes();
     }
+    #endregion
+
+    #region tree similarity
+    public static double ComputeSimilarity(ISymbolicExpressionTree t1, ISymbolicExpressionTree t2, bool simplify = false, bool strict = false) {
+      return ComputeSimilarity(t1.ActualRoot(), t2.ActualRoot(), simplify, strict);
+    }
+
+    public static double ComputeSimilarity(ISymbolicExpressionTreeNode t1, ISymbolicExpressionTreeNode t2, bool simplify = false, bool strict = false) {
+      var lh = t1.Hash(simplify, strict);
+      var rh = t2.Hash(simplify, strict);
+
+      Array.Sort(lh);
+      Array.Sort(rh);
+
+      return ComputeSimilarity(lh, rh);
+    }
+
+    // requires lhs and rhs to be sorted
+    public static int IntersectCount(this ulong[] lh, ulong[] rh) {
+      int count = 0;
+      for (int i = 0, j = 0; i < lh.Length && j < rh.Length;) {
+        var h1 = lh[i];
+        var h2 = rh[j];
+        if (h1 == h2) {
+          ++count;
+          ++i;
+          ++j;
+        } else if (h1 < h2) {
+          ++i;
+        } else if (h1 > h2) {
+          ++j;
+        }
+      }
+      return count;
+    }
+
+    public static IEnumerable<ulong> Intersect(this ulong[] lh, ulong[] rh) {
+      for (int i = 0, j = 0; i < lh.Length && j < rh.Length;) {
+        var h1 = lh[i];
+        var h2 = rh[j];
+        if (h1 == h2) {
+          yield return h1;
+          ++i;
+          ++j;
+        } else if (h1 < h2) {
+          ++i;
+        } else if (h1 > h2) {
+          ++j;
+        }
+      }
+    }
+
+    // this will only work if lh and rh are sorted
+    public static double ComputeSimilarity(ulong[] lh, ulong[] rh) {
+      return 2d * IntersectCount(lh, rh) / (lh.Length + rh.Length);
+    }
+
+    public static double ComputeAverageSimilarity(IList<ISymbolicExpressionTree> trees, bool simplify = false, bool strict = false) {
+      var total = trees.Count * (trees.Count - 1) / 2;
+      double avg = 0;
+      var hashes = new ulong[trees.Count][];
+      // build hash arrays
+      for (int i = 0; i < trees.Count; ++i) {
+        var nodes = trees[i].MakeNodes(strict);
+        hashes[i] = (simplify ? nodes.Simplify(HashUtil.DJBHash) : nodes.Sort(HashUtil.DJBHash)).Select(x => x.CalculatedHashValue).ToArray();
+        Array.Sort(hashes[i]);
+      }
+      // compute similarity matrix
+      for (int i = 0; i < trees.Count - 1; ++i) {
+        for (int j = i + 1; j < trees.Count; ++j) {
+          avg += ComputeSimilarity(hashes[i], hashes[j]);
+        }
+      }
+      return avg / total;
+    }
+
+    public static double[,] ComputeSimilarityMatrix(IList<ISymbolicExpressionTree> trees, bool simplify = false, bool strict = false) {
+      var sim = new double[trees.Count, trees.Count];
+      var hashes = new ulong[trees.Count][];
+      // build hash arrays
+      for (int i = 0; i < trees.Count; ++i) {
+        var nodes = trees[i].MakeNodes(strict);
+        hashes[i] = (simplify ? nodes.Simplify(HashUtil.DJBHash) : nodes.Sort(HashUtil.DJBHash)).Select(x => x.CalculatedHashValue).ToArray();
+        Array.Sort(hashes[i]);
+      }
+      // compute similarity matrix
+      for (int i = 0; i < trees.Count - 1; ++i) {
+        for (int j = i + 1; j < trees.Count; ++j) {
+          sim[i, j] = sim[j, i] = ComputeSimilarity(hashes[i], hashes[j]);
+        }
+      }
+      return sim;
+    }
+    #endregion
 
     #region parse a nodes array back into a tree
@@ -218 +243 @@
     #region tree simplification
     // these simplification methods rely on the assumption that child nodes of the current node have already been simplified
-    // (in other words simplification should be applied in a bottom-up fashion) 
+    // (in other words simplification should be applied in a bottom-up fashion)
     public static ISymbolicExpressionTree Simplify(ISymbolicExpressionTree tree) {
       ulong hashFunction(byte[] bytes) => HashUtil.JSHash(bytes);
@@ -248 +273 @@
     }
 
-    // simplify multiplications by reducing constants and div terms  
+    // simplify multiplications by reducing constants and div terms
     public static void SimplifyMultiplication(ref HashNode<ISymbolicExpressionTreeNode>[] nodes, int i) {
       var node = nodes[i];