Changeset 16353

Timestamp:

12/08/18 07:52:35 (6 years ago)

Author:

gkronber

Message:

#2937: added partial support for rectifier, leaky-rectifier and soft-plus functions

Location:

branches/2937_SymReg_AnalyticalQuotient/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4

Files:

• Converters/TreeToAutoDiffTermConverter.cs (modified) (3 diffs)
• Grammars/FullFunctionalExpressionGrammar.cs (modified) (2 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj (modified) (1 diff)
• Interpreter/OpCodes.cs (modified) (2 diffs)
• Interpreter/SymbolicDataAnalysisExpressionTreeLinearInterpreter.cs (modified) (1 diff)
• Symbols/LeakyRectifier.cs (added)
• Symbols/Rectifier.cs (added)
• Symbols/SoftSum.cs (added)

branches/2937_SymReg_AnalyticalQuotient/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Converters/TreeToAutoDiffTermConverter.cs

@@ -85 +85 @@
       diff: x => -(Math.Exp(-(x * x)) * Math.Sqrt(Math.Exp(x * x)) * x) / Math.Sqrt(2 * Math.PI));
 
+    private static readonly Func<Term, UnaryFunc> rectifier = UnaryFunc.Factory(
+      eval: x => Math.Max(x, 0),
+      diff: x => x > 0 ? 1.0 : 0.0);
+
+    private static readonly Func<Term, UnaryFunc> leakyRectifier = UnaryFunc.Factory(
+      eval: x => Math.Max(x, 0.01 * x),
+      diff: x => x > 0 ? 1.0 : 0.01);
+
     #endregion
 
@@ -254 +262 @@
           ConvertToAutoDiff(node.GetSubtree(0)));
       }
+      if (node.Symbol is Rectifier) {
+        return rectifier(ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is LeakyRectifier) {
+        return leakyRectifier(ConvertToAutoDiff(node.GetSubtree(0)));
+      }
+      if (node.Symbol is Softplus) {
+        return TermBuilder.Log(1.0 + TermBuilder.Exp(ConvertToAutoDiff(node.GetSubtree(0))));
+      }
       if (node.Symbol is StartSymbol) {
         if (addLinearScalingTerms) {
@@ -296 +313 @@
           !(n.Symbol is Subtraction) &&
           !(n.Symbol is Multiplication) &&
+          !(n.Symbol is AnalyticalQuotient) &&
+          !(n.Symbol is Rectifier) &&
+          !(n.Symbol is LeakyRectifier) &&
+          !(n.Symbol is Softplus) &&
           !(n.Symbol is Division) &&
           !(n.Symbol is Logarithm) &&

branches/2937_SymReg_AnalyticalQuotient/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Grammars/FullFunctionalExpressionGrammar.cs

@@ -93 +93 @@
       sineIntegral.InitialFrequency = 0.0;
 
+      var rect = new Rectifier();
+      var softplus = new Softplus();
+      var leakyRect = new LeakyRectifier();
+
       var exp = new Exponential();
       var @if = new IfThenElse();
@@ -126 +130 @@
       var allSymbols = new List<Symbol>() { add, sub, mul, div, aq, mean, sin, cos, tan, log, square, pow, sqrt, root, exp,
         airyA, airyB, bessel, cosineIntegral, dawson, erf, expIntegralEi, fresnelCosineIntegral, fresnelSineIntegral, gamma, hypCosineIntegral, hypSineIntegral, norm, psi, sineIntegral,
-        @if, gt, lt, and, or, not,xor, timeLag, integral, derivative, constant, variableSymbol, binFactorVariable, factorVariable, laggedVariable,autoregressiveVariable, variableCondition };
+        @if, gt, lt, and, or, not,xor, timeLag, integral, derivative, constant, variableSymbol, binFactorVariable, factorVariable, laggedVariable,autoregressiveVariable, variableCondition,
+      softplus, rect, leakyRect};
       var unaryFunctionSymbols = new List<Symbol>() { square, sqrt, sin, cos, tan, log, exp, not, timeLag, integral, derivative,
         airyA, airyB, bessel, cosineIntegral, dawson, erf, expIntegralEi, fresnelCosineIntegral, fresnelSineIntegral, gamma, hypCosineIntegral, hypSineIntegral, norm, psi, sineIntegral
+        , rect, softplus, leakyRect
       };
 

branches/2937_SymReg_AnalyticalQuotient/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.csproj

@@ -198 +198 @@
     <Compile Include="SymbolicDataAnalysisSolutionImpactValuesCalculator.cs" />
     <Compile Include="Symbols\Addition.cs" />
+    <Compile Include="Symbols\LeakyRectifier.cs" />
+    <Compile Include="Symbols\Rectifier.cs" />
+    <Compile Include="Symbols\SoftSum.cs" />
     <Compile Include="Symbols\And.cs" />
     <Compile Include="Symbols\AutoregressiveVariable.cs" />

branches/2937_SymReg_AnalyticalQuotient/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/OpCodes.cs

@@ -86 +86 @@
     public const byte BinaryFactorVariable = 47;
     public const byte AnalyticalQuotient = 48;
+    public const byte Rectifier = 49;
+    public const byte Softplus = 50;
+    public const byte LeakyRectifier = 51;
 
 
@@ -137 +140 @@
       { typeof(FactorVariable), OpCodes.FactorVariable },
       { typeof(BinaryFactorVariable), OpCodes.BinaryFactorVariable },
-      {typeof(AnalyticalQuotient), OpCodes.AnalyticalQuotient }
+      {typeof(AnalyticalQuotient), OpCodes.AnalyticalQuotient },
+      {typeof(Rectifier), OpCodes.Rectifier },
+      {typeof(Softplus), OpCodes.Softplus},
+      {typeof(LeakyRectifier), OpCodes.LeakyRectifier },
     };
 

branches/2937_SymReg_AnalyticalQuotient/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/SymbolicDataAnalysisExpressionTreeLinearInterpreter.cs

@@ -346 +346 @@
           }
           instr.value = result;
+        } else if (instr.opCode == OpCodes.Rectifier) {
+          return Math.Max(code[instr.childIndex].value, 0.0);
+        } else if (instr.opCode == OpCodes.LeakyRectifier) {
+          var x = code[instr.childIndex].value;
+          if (x > 0) return x; else return 0.01 * x;
+        } else if (instr.opCode == OpCodes.Softplus) {
+          var x = code[instr.childIndex].value;
+          return Math.Log(1.0 + Math.Exp(x));
         } else if (instr.opCode == OpCodes.AND) {
           double result = code[instr.childIndex].value;