Changeset 18146 for branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis

Timestamp:

12/16/21 14:23:48 (3 years ago)

Author:

mkommend

Message:

#3136: Merged trunk changes into branch.

Location:

branches/3136_Structural_GP

Files:

• . (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4 (modified) (1 prop)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs (modified) (1 diff)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs (modified) (6 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs (modified) (3 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelAlglib_3_7.cs (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelFull.cs (modified) (4 diffs)
• HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs (modified) (1 diff)

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/GBM/GradientBoostingRegressionAlgorithm.cs

@@ -417 +417 @@
       var addNode = new Addition().CreateTreeNode();
       var mulNode = new Multiplication().CreateTreeNode();
-      var scaleNode = (ConstantTreeNode)new Constant().CreateTreeNode(); // all models are scaled using the same nu
+      var scaleNode = (NumberTreeNode)new Number().CreateTreeNode(); // all models are scaled using the same nu
       scaleNode.Value = nu;
 

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/GaussianProcess/GaussianProcessCovarianceOptimizationProblem.cs

@@ -97 +97 @@
 
     private const string ProblemDataParameterName = "ProblemData";
-    private const string ConstantOptIterationsParameterName = "Constant optimization steps";
+    private const string ParameterOptIterationsParameterName = "Parameter optimization steps";
     private const string RestartsParameterName = "Restarts";
 
@@ -108 +108 @@
       get { return (IValueParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
     }
-    public IFixedValueParameter<IntValue> ConstantOptIterationsParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters[ConstantOptIterationsParameterName]; }
+    public IFixedValueParameter<IntValue> ParameterOptIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[ParameterOptIterationsParameterName]; }
     }
     public IFixedValueParameter<IntValue> RestartsParameter {
@@ -124 +124 @@
     IDataAnalysisProblemData IDataAnalysisProblem.ProblemData { get { return ProblemData; } }
 
-    public int ConstantOptIterations {
-      get { return ConstantOptIterationsParameter.Value.Value; }
-      set { ConstantOptIterationsParameter.Value.Value = value; }
+    public int ParameterOptIterations {
+      get { return ParameterOptIterationsParameter.Value.Value; }
+      set { ParameterOptIterationsParameter.Value.Value = value; }
     }
 
@@ -153 +153 @@
       : base() {
       Parameters.Add(new ValueParameter<IRegressionProblemData>(ProblemDataParameterName, "The data for the regression problem", new RegressionProblemData()));
-      Parameters.Add(new FixedValueParameter<IntValue>(ConstantOptIterationsParameterName, "Number of optimization steps for hyperparameter values", new IntValue(50)));
-      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of random restarts for constant optimization.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(ParameterOptIterationsParameterName, "Number of optimization steps for hyperparameter values", new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of random restarts for parameter optimization.", new IntValue(10)));
       Parameters["Restarts"].Hidden = true;
       var g = new SimpleSymbolicExpressionGrammar();
@@ -236 +236 @@
         double epsx = 0;
         double stpmax = 1;
-        int maxits = ConstantOptIterations;
+        int maxits = ParameterOptIterations;
         alglib.mincgstate state;
         alglib.mincgreport rep;
@@ -384 +384 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(ParameterOptIterationsParameterName)) {
+        if (Parameters.ContainsKey("Constant optimization steps")) {
+          Parameters.Add(new FixedValueParameter<IntValue>(ParameterOptIterationsParameterName, "Number of optimization steps for hyperparameter values", (IntValue)Parameters["Constant optimization steps"].ActualValue));
+          Parameters.Remove("Constant optimization steps");
+        } else {
+          Parameters.Add(new FixedValueParameter<IntValue>(ParameterOptIterationsParameterName, "Number of optimization steps for hyperparameter values", new IntValue(50)));
+        }
+      }
     }
 

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/GradientBoostedTrees/RegressionTreeModel.cs

@@ -230 +230 @@
       var startSy = new StartSymbol();
       var varCondSy = new VariableCondition() { IgnoreSlope = true };
-      var constSy = new Constant();
+      var numSy = new Number();
 
       var startNode = startSy.CreateTreeNode();
-      startNode.AddSubtree(CreateSymbolicRegressionTreeRecursive(tree, 0, varCondSy, constSy));
+      startNode.AddSubtree(CreateSymbolicRegressionTreeRecursive(tree, 0, varCondSy, numSy));
       var rootNode = rootSy.CreateTreeNode();
       rootNode.AddSubtree(startNode);
@@ -239 +239 @@
     }
 
-    private ISymbolicExpressionTreeNode CreateSymbolicRegressionTreeRecursive(TreeNode[] treeNodes, int nodeIdx, VariableCondition varCondSy, Constant constSy) {
+    private ISymbolicExpressionTreeNode CreateSymbolicRegressionTreeRecursive(TreeNode[] treeNodes, int nodeIdx, VariableCondition varCondSy, Number numSy) {
       var curNode = treeNodes[nodeIdx];
       if (curNode.VarName == TreeNode.NO_VARIABLE) {
-        var node = (ConstantTreeNode)constSy.CreateTreeNode();
+        var node = (NumberTreeNode)numSy.CreateTreeNode();
         node.Value = curNode.Val;
         return node;
@@ -250 +250 @@
         node.Threshold = curNode.Val;
 
-        var left = CreateSymbolicRegressionTreeRecursive(treeNodes, curNode.LeftIdx, varCondSy, constSy);
-        var right = CreateSymbolicRegressionTreeRecursive(treeNodes, curNode.RightIdx, varCondSy, constSy);
+        var left = CreateSymbolicRegressionTreeRecursive(treeNodes, curNode.LeftIdx, varCondSy, numSy);
+        var right = CreateSymbolicRegressionTreeRecursive(treeNodes, curNode.RightIdx, varCondSy, numSy);
         node.AddSubtree(left);
         node.AddSubtree(right);

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/NonlinearRegression/NonlinearRegression.cs

@@ -123 +123 @@
       : base() {
       Problem = new RegressionProblem();
-      Parameters.Add(new FixedValueParameter<StringValue>(ModelStructureParameterName, "The function for which the parameters must be fit (only numeric constants are tuned).", new StringValue("1.0 * x*x + 0.0")));
-      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "The maximum number of iterations for constants optimization.", new IntValue(200)));
+      Parameters.Add(new FixedValueParameter<StringValue>(ModelStructureParameterName, 
+        "The expression for which the <num> parameters should be fit.\n " +
+        "Defined constants will not be modified.\n " +
+        "Modifiable numbers are specified with <num>. To specify a default value within this number symbol, a default value can be declared by e.g. <num=1.0>.", 
+        new StringValue("<num> * x*x + 0.0")));
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "The maximum number of iterations for parameter optimization.", new IntValue(200)));
       Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of independent random restarts (>0)", new IntValue(10)));
       Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The PRNG seed value.", new IntValue()));
@@ -210 +214 @@
 
     /// <summary>
-    /// Fits a model to the data by optimizing the numeric constants.
+    /// Fits a model to the data by optimizing parameters.
     /// Model is specified as infix expression containing variable names and numbers. 
-    /// The starting point for the numeric constants is initialized randomly if a random number generator is specified (~N(0,1)). Otherwise the user specified constants are
+    /// The starting values for the parameters are initialized randomly if a random number generator is specified (~N(0,1)). Otherwise the user specified values are
     /// used as a starting point. 
     /// </summary>-
     /// <param name="problemData">Training and test data</param>
     /// <param name="modelStructure">The function as infix expression</param>
-    /// <param name="maxIterations">Number of constant optimization iterations (using Levenberg-Marquardt algorithm)</param>
-    /// <param name="random">Optional random number generator for random initialization of numeric constants.</param>
+    /// <param name="maxIterations">Number of Levenberg-Marquardt iterations</param>
+    /// <param name="random">Optional random number generator for random initialization of parameters.</param>
     /// <returns></returns>
     public static ISymbolicRegressionSolution CreateRegressionSolution(IRegressionProblemData problemData, string modelStructure, int maxIterations, bool applyLinearScaling, IRandom rand = null) {
@@ -263 +267 @@
       }
 
-      if (!SymbolicRegressionConstantOptimizationEvaluator.CanOptimizeConstants(tree)) throw new ArgumentException("The optimizer does not support the specified model structure.");
-
-      // initialize constants randomly
+      if (!SymbolicRegressionParameterOptimizationEvaluator.CanOptimizeParameters(tree)) throw new ArgumentException("The optimizer does not support the specified model structure.");
+
+      // initialize parameters randomly
       if (rand != null) {
-        foreach (var node in tree.IterateNodesPrefix().OfType<ConstantTreeNode>()) {
+        foreach (var node in tree.IterateNodesPrefix().OfType<NumberTreeNode>()) {
           double f = Math.Exp(NormalDistributedRandom.NextDouble(rand, 0, 1));
           double s = rand.NextDouble() < 0.5 ? -1 : 1;
@@ -275 +279 @@
       var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
 
-      SymbolicRegressionConstantOptimizationEvaluator.OptimizeConstants(interpreter, tree, problemData, problemData.TrainingIndices,
+      SymbolicRegressionParameterOptimizationEvaluator.OptimizeParameters(interpreter, tree, problemData, problemData.TrainingIndices,
         applyLinearScaling: applyLinearScaling, maxIterations: maxIterations,
-        updateVariableWeights: false, updateConstantsInTree: true);
+        updateVariableWeights: false, updateParametersInTree: true);
 
       var model = new SymbolicRegressionModel(problemData.TargetVariable, tree, (ISymbolicDataAnalysisExpressionTreeInterpreter)interpreter.Clone());

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModel.cs

@@ -220 +220 @@
       }
 
-      var constSy = new Constant();
+      var numSy = new Number();
       var varCondSy = new VariableCondition() { IgnoreSlope = true };
 
-      var node = CreateRegressionTreeRec(rf.innerobj.trees, offset, offset + 1, constSy, varCondSy);
+      var node = CreateRegressionTreeRec(rf.innerobj.trees, offset, offset + 1, numSy, varCondSy);
 
       var startNode = new StartSymbol().CreateTreeNode();
@@ -232 +232 @@
     }
 
-    private ISymbolicExpressionTreeNode CreateRegressionTreeRec(double[] trees, int offset, int k, Constant constSy, VariableCondition varCondSy) {
+    private ISymbolicExpressionTreeNode CreateRegressionTreeRec(double[] trees, int offset, int k, Number numSy, VariableCondition varCondSy) {
 
       // alglib source for evaluation of one tree (dfprocessinternal)
@@ -262 +262 @@
 
       if ((double)(trees[k]) == (double)(-1)) {
-        var constNode = (ConstantTreeNode)constSy.CreateTreeNode();
-        constNode.Value = trees[k + 1];
-        return constNode;
+        var numNode = (NumberTreeNode)numSy.CreateTreeNode();
+        numNode.Value = trees[k + 1];
+        return numNode;
       } else {
         var condNode = (VariableConditionTreeNode)varCondSy.CreateTreeNode();
@@ -271 +271 @@
         condNode.Slope = double.PositiveInfinity;
 
-        var left = CreateRegressionTreeRec(trees, offset, k + 3, constSy, varCondSy);
-        var right = CreateRegressionTreeRec(trees, offset, offset + (int)Math.Round(trees[k + 2]), constSy, varCondSy);
+        var left = CreateRegressionTreeRec(trees, offset, k + 3, numSy, varCondSy);
+        var right = CreateRegressionTreeRec(trees, offset, offset + (int)Math.Round(trees[k + 2]), numSy, varCondSy);
 
         condNode.AddSubtree(left); // not 100% correct because interpreter uses: if(x <= thres) left() else right() and RF uses if(x < thres) left() else right() (see above)

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelAlglib_3_7.cs

@@ -220 +220 @@
       }
 
-      var constSy = new Constant();
+      var numSy = new Number();
       var varCondSy = new VariableCondition() { IgnoreSlope = true };
 
-      var node = CreateRegressionTreeRec(rf.innerobj.trees, offset, offset + 1, constSy, varCondSy);
+      var node = CreateRegressionTreeRec(rf.innerobj.trees, offset, offset + 1, numSy, varCondSy);
 
       var startNode = new StartSymbol().CreateTreeNode();
@@ -232 +232 @@
     }
 
-    private ISymbolicExpressionTreeNode CreateRegressionTreeRec(double[] trees, int offset, int k, Constant constSy, VariableCondition varCondSy) {
+    private ISymbolicExpressionTreeNode CreateRegressionTreeRec(double[] trees, int offset, int k, Number numSy, VariableCondition varCondSy) {
 
       // alglib source for evaluation of one tree (dfprocessinternal)
@@ -261 +261 @@
       // }
 
-      if ((double)(trees[k]) == (double)(-1)) {
-        var constNode = (ConstantTreeNode)constSy.CreateTreeNode();
-        constNode.Value = trees[k + 1];
-        return constNode;
+      if (trees[k] == -1) {
+        var numNode = (NumberTreeNode)numSy.CreateTreeNode();
+        numNode.Value = trees[k + 1];
+        return numNode;
       } else {
         var condNode = (VariableConditionTreeNode)varCondSy.CreateTreeNode();
@@ -271 +271 @@
         condNode.Slope = double.PositiveInfinity;
 
-        var left = CreateRegressionTreeRec(trees, offset, k + 3, constSy, varCondSy);
-        var right = CreateRegressionTreeRec(trees, offset, offset + (int)Math.Round(trees[k + 2]), constSy, varCondSy);
+        var left = CreateRegressionTreeRec(trees, offset, k + 3, numSy, varCondSy);
+        var right = CreateRegressionTreeRec(trees, offset, offset + (int)Math.Round(trees[k + 2]), numSy, varCondSy);
 
         condNode.AddSubtree(left); // not 100% correct because interpreter uses: if(x <= thres) left() else right() and RF uses if(x < thres) left() else right() (see above)

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/RandomForest/RandomForestModelFull.cs

@@ -199 +199 @@
       }
 
-      var constSy = new Constant();
+      var numSy = new Number();
       var varCondSy = new VariableCondition() { IgnoreSlope = true };
 
-      var node = CreateRegressionTreeRec(rf.innerobj.trees, offset, offset + 1, constSy, varCondSy);
+      var node = CreateRegressionTreeRec(rf.innerobj.trees, offset, offset + 1, numSy, varCondSy);
 
       var startNode = new StartSymbol().CreateTreeNode();
@@ -211 +211 @@
     }
 
-    private ISymbolicExpressionTreeNode CreateRegressionTreeRec(double[] trees, int offset, int k, Constant constSy, VariableCondition varCondSy) {
+    private ISymbolicExpressionTreeNode CreateRegressionTreeRec(double[] trees, int offset, int k, Number numSy, VariableCondition varCondSy) {
 
       // alglib source for evaluation of one tree (dfprocessinternal)
@@ -240 +240 @@
       // }
 
-      if ((double)(trees[k]) == (double)(-1)) {
-        var constNode = (ConstantTreeNode)constSy.CreateTreeNode();
-        constNode.Value = trees[k + 1];
-        return constNode;
+      if (trees[k] == -1) {
+        var numNode = (NumberTreeNode)numSy.CreateTreeNode();
+        numNode.Value = trees[k + 1];
+        return numNode;
       } else {
         var condNode = (VariableConditionTreeNode)varCondSy.CreateTreeNode();
@@ -250 +250 @@
         condNode.Slope = double.PositiveInfinity;
 
-        var left = CreateRegressionTreeRec(trees, offset, k + 3, constSy, varCondSy);
-        var right = CreateRegressionTreeRec(trees, offset, offset + (int)Math.Round(trees[k + 2]), constSy, varCondSy);
+        var left = CreateRegressionTreeRec(trees, offset, k + 3, numSy, varCondSy);
+        var right = CreateRegressionTreeRec(trees, offset, offset + (int)Math.Round(trees[k + 2]), numSy, varCondSy);
 
         condNode.AddSubtree(left); // not 100% correct because interpreter uses: if(x <= thres) left() else right() and RF uses if(x < thres) left() else right() (see above)

branches/3136_Structural_GP/HeuristicLab.Algorithms.DataAnalysis/3.4/TimeSeries/AutoregressiveModeling.cs

@@ -104 +104 @@
       int nRows = inputMatrix.GetLength(0);
       int nFeatures = inputMatrix.GetLength(1) - 1;
-      double[] coefficients = new double[nFeatures + 1]; // last coefficient is for the constant
+      double[] coefficients = new double[nFeatures + 1]; // last coefficient is for the offset
 
       int retVal = 1;