Changeset 17602

Timestamp:

06/16/20 11:21:34 (4 years ago)

Author:

pfleck

Message:

#3040

• Changed stddev, variance, etc. to population variant
• Added multiplicative simplifications for stdev and variance symbols

Location:

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4

Files:

• Converters/VectorTreeSimplifier.cs (modified) (5 diffs)
• Interpreter/SymbolicDataAnalysisExpressionTreeVectorInterpreter.cs (modified) (3 diffs)

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/VectorTreeSimplifier.cs

@@ -62 +62 @@
     private static readonly Mean meanSymbol = new Mean();
     private static readonly Length lengthSymbol = new Length();
+    private static readonly StandardDeviation standardDeviationSymbol = new StandardDeviation();
+    private static readonly Variance varianceSymbol = new Variance();
 
     private readonly SymbolicDataAnalysisExpressionTreeVectorInterpreter interpreter;
@@ -271 +273 @@
     private static bool IsLength(ISymbolicExpressionTreeNode node) {
       return node.Symbol is Length;
+    }
+
+    private static bool IsStandardDeviation(ISymbolicExpressionTreeNode node) {
+      return node.Symbol is StandardDeviation;
+    }
+
+    private static bool IsVariance(ISymbolicExpressionTreeNode node) {
+      return node.Symbol is Variance;
     }
     #endregion
@@ -351 +361 @@
       } else if (IsLength(original)) {
         return SimplifyLengthAggregation(original);
+      } else if (IsStandardDeviation(original)) {
+        return SimplifyStandardDeviationAggregation(original);
+      } else if (IsVariance(original)) {
+        return SimplifyVarianceAggregation(original);
       } else {
         return SimplifyAny(original);
@@ -554 +568 @@
     private ISymbolicExpressionTreeNode SimplifyLengthAggregation(ISymbolicExpressionTreeNode original) {
       return MakeLengthAggregation(GetSimplifiedTree(original.GetSubtree(0)));
+    }
+
+    private ISymbolicExpressionTreeNode SimplifyStandardDeviationAggregation(ISymbolicExpressionTreeNode original) {
+      return MakeStandardDeviationAggregation(GetSimplifiedTree(original.GetSubtree(0)));
+    }
+
+    private ISymbolicExpressionTreeNode SimplifyVarianceAggregation(ISymbolicExpressionTreeNode original) {
+      return MakeVarianceAggregation(GetSimplifiedTree(original.GetSubtree(0)));
     }
     #endregion
@@ -1561 +1583 @@
     }
 
+    private ISymbolicExpressionTreeNode MakeStandardDeviationAggregation(ISymbolicExpressionTreeNode node) {
+      if (IsConstant(node)) {
+        return MakeConstant(0.0);
+      } else if (IsMultiplication(node) || IsDivision(node)) {
+        var factors = node.Subtrees;
+        if (IsDivision(node)) factors = InvertNodes(factors, Invert);
+
+        var scalarFactors = factors.Where(IsScalarNode).ToList();
+        var remainingFactors = factors.Except(scalarFactors).ToList();
+
+        if (scalarFactors.Any() && remainingFactors.Any()) {
+          var scalarNode = scalarFactors.Aggregate(MakeProduct);
+          var vectorNode = remainingFactors.Aggregate(MakeProduct);
+
+          var stdevNode = MakeStandardDeviationAggregation(vectorNode);
+
+          return MakeProduct(scalarNode, stdevNode);
+        } else if (scalarFactors.Any()) {
+          var scalarNode = scalarFactors.Aggregate(MakeProduct);
+          return scalarNode;
+        } else if (remainingFactors.Any()) {
+          var vectorNode = remainingFactors.Aggregate(MakeProduct);
+          var stdevNode = standardDeviationSymbol.CreateTreeNode();
+          stdevNode.AddSubtree(vectorNode);
+          return stdevNode;
+        } else
+          throw new InvalidOperationException("Multiplication does not contain any terms to simplify.");
+      } else if (IsVariableBase(node)) { // weight is like multiplication
+        var variableNode = (VariableTreeNodeBase)node;
+        var weight = variableNode.Weight;
+        variableNode.Weight = 1.0;
+        var stdevNode = standardDeviationSymbol.CreateTreeNode();
+        stdevNode.AddSubtree(node);
+        return MakeProduct(MakeConstant(weight), stdevNode);
+      } else {
+        var stdevNode = standardDeviationSymbol.CreateTreeNode();
+        stdevNode.AddSubtree(node);
+        return stdevNode;
+      }
+    }
+
+    private ISymbolicExpressionTreeNode MakeVarianceAggregation(ISymbolicExpressionTreeNode node) {
+      if (IsConstant(node)) {
+        return MakeConstant(0.0);
+      } else if (IsMultiplication(node) || IsDivision(node)) {
+        var factors = node.Subtrees;
+        if (IsDivision(node)) factors = InvertNodes(factors, Invert);
+
+        var scalarFactors = factors.Where(IsScalarNode).ToList();
+        var remainingFactors = factors.Except(scalarFactors).ToList();
+
+        if (scalarFactors.Any() && remainingFactors.Any()) {
+          var scalarNode = scalarFactors.Aggregate(MakeProduct);
+          var vectorNode = remainingFactors.Aggregate(MakeProduct);
+
+          var varNode = MakeVarianceAggregation(vectorNode);
+
+          return MakeProduct(MakeSquare(scalarNode), varNode);
+        } else if (scalarFactors.Any()) {
+          var scalarNode = scalarFactors.Aggregate(MakeProduct);
+          return MakeSquare(scalarNode);
+        } else if (remainingFactors.Any()) {
+          var vectorNode = remainingFactors.Aggregate(MakeProduct);
+          var varNode = varianceSymbol.CreateTreeNode();
+          varNode.AddSubtree(vectorNode);
+          return varNode;
+        } else
+          throw new InvalidOperationException("Multiplication does not contain any terms to simplify.");
+      } else if (IsVariableBase(node)) { // weight is like multiplication
+        var variableNode = (VariableTreeNodeBase)node;
+        var weight = variableNode.Weight;
+        variableNode.Weight = 1.0;
+        var varNode = varianceSymbol.CreateTreeNode();
+        varNode.AddSubtree(node);
+        return MakeProduct(MakeSquare(MakeConstant(weight)), varNode);
+      } else {
+        var varNode = varianceSymbol.CreateTreeNode();
+        varNode.AddSubtree(node);
+        return varNode;
+      }
+    }
     #endregion
 

branches/3040_VectorBasedGP/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeVectorInterpreter.cs

@@ -415 +415 @@
             var cur = Evaluate(dataset, ref row, state);
             return AggregateApply(cur,
-              s => double.NaN,
-              v => Statistics.StandardDeviation(v));
+              s => 0,
+              v => Statistics.PopulationStandardDeviation(v));
           }
         case OpCodes.Length: {
@@ -439 +439 @@
             var cur = Evaluate(dataset, ref row, state);
             return AggregateApply(cur,
+              s => 0,
+              v => Statistics.PopulationVariance(v));
+          }
+        case OpCodes.Skewness: {
+            var cur = Evaluate(dataset, ref row, state);
+            return AggregateApply(cur,
               s => double.NaN,
-              v => Statistics.Variance(v));
-          }
-        case OpCodes.Skewness: {
+              v => Statistics.PopulationSkewness(v));
+          }
+        case OpCodes.Kurtosis: {
             var cur = Evaluate(dataset, ref row, state);
             return AggregateApply(cur,
               s => double.NaN,
-              v => Statistics.Skewness(v));
-          }
-        case OpCodes.Kurtosis: {
-            var cur = Evaluate(dataset, ref row, state);
-            return AggregateApply(cur,
-              s => double.NaN,
-              v => Statistics.Kurtosis(v));
+              v => Statistics.PopulationKurtosis(v));
           }
         case OpCodes.EuclideanDistance: {
@@ -470 +470 @@
               //(s1, v2) => 0,
               //(v1, s2) => 0,
-              vvFunc: (v1, v2) => v1.Count == v2.Count ? Statistics.Covariance(v1, v2) : double.NaN);
+              vvFunc: (v1, v2) => v1.Count == v2.Count ? Statistics.PopulationCovariance(v1, v2) : double.NaN);
           }
         case OpCodes.Variable: {