[16912] | 1 | #region License Information
|
---|
| 2 | /* HeuristicLab
|
---|
| 3 | * Copyright (C) 2002-2019 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
| 4 | *
|
---|
| 5 | * This file is part of HeuristicLab.
|
---|
| 6 | *
|
---|
| 7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
| 8 | * it under the terms of the GNU General Public License as published by
|
---|
| 9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
| 10 | * (at your option) any later version.
|
---|
| 11 | *
|
---|
| 12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
| 15 | * GNU General Public License for more details.
|
---|
| 16 | *
|
---|
| 17 | * You should have received a copy of the GNU General Public License
|
---|
| 18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
| 19 | */
|
---|
| 20 | #endregion
|
---|
| 21 |
|
---|
| 22 | using System;
|
---|
| 23 | using System.Collections.Generic;
|
---|
| 24 | using System.Linq;
|
---|
| 25 | using HeuristicLab.Common;
|
---|
| 26 | using HeuristicLab.Core;
|
---|
| 27 | using HeuristicLab.Data;
|
---|
| 28 | using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
|
---|
| 29 | using HeuristicLab.Optimization;
|
---|
| 30 | using HeuristicLab.Parameters;
|
---|
| 31 | using HEAL.Attic;
|
---|
| 32 |
|
---|
| 33 | namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
|
---|
[16914] | 34 | [Item("Constant Optimization Evaluator (with constraints)", "")]
|
---|
| 35 | [StorableType("A8958E06-C54A-4193-862E-8315C86EB5C1")]
|
---|
| 36 | public class ConstrainedConstantOptimizationEvaluator : SymbolicRegressionSingleObjectiveEvaluator {
|
---|
[16912] | 37 | private const string ConstantOptimizationIterationsParameterName = "ConstantOptimizationIterations";
|
---|
| 38 | private const string ConstantOptimizationImprovementParameterName = "ConstantOptimizationImprovement";
|
---|
| 39 | private const string ConstantOptimizationProbabilityParameterName = "ConstantOptimizationProbability";
|
---|
| 40 | private const string ConstantOptimizationRowsPercentageParameterName = "ConstantOptimizationRowsPercentage";
|
---|
| 41 | private const string UpdateConstantsInTreeParameterName = "UpdateConstantsInSymbolicExpressionTree";
|
---|
| 42 | private const string UpdateVariableWeightsParameterName = "Update Variable Weights";
|
---|
| 43 |
|
---|
| 44 | private const string FunctionEvaluationsResultParameterName = "Constants Optimization Function Evaluations";
|
---|
| 45 | private const string GradientEvaluationsResultParameterName = "Constants Optimization Gradient Evaluations";
|
---|
| 46 | private const string CountEvaluationsParameterName = "Count Function and Gradient Evaluations";
|
---|
| 47 |
|
---|
| 48 | public IFixedValueParameter<IntValue> ConstantOptimizationIterationsParameter {
|
---|
| 49 | get { return (IFixedValueParameter<IntValue>)Parameters[ConstantOptimizationIterationsParameterName]; }
|
---|
| 50 | }
|
---|
| 51 | public IFixedValueParameter<DoubleValue> ConstantOptimizationImprovementParameter {
|
---|
| 52 | get { return (IFixedValueParameter<DoubleValue>)Parameters[ConstantOptimizationImprovementParameterName]; }
|
---|
| 53 | }
|
---|
| 54 | public IFixedValueParameter<PercentValue> ConstantOptimizationProbabilityParameter {
|
---|
| 55 | get { return (IFixedValueParameter<PercentValue>)Parameters[ConstantOptimizationProbabilityParameterName]; }
|
---|
| 56 | }
|
---|
| 57 | public IFixedValueParameter<PercentValue> ConstantOptimizationRowsPercentageParameter {
|
---|
| 58 | get { return (IFixedValueParameter<PercentValue>)Parameters[ConstantOptimizationRowsPercentageParameterName]; }
|
---|
| 59 | }
|
---|
| 60 | public IFixedValueParameter<BoolValue> UpdateConstantsInTreeParameter {
|
---|
| 61 | get { return (IFixedValueParameter<BoolValue>)Parameters[UpdateConstantsInTreeParameterName]; }
|
---|
| 62 | }
|
---|
| 63 | public IFixedValueParameter<BoolValue> UpdateVariableWeightsParameter {
|
---|
| 64 | get { return (IFixedValueParameter<BoolValue>)Parameters[UpdateVariableWeightsParameterName]; }
|
---|
| 65 | }
|
---|
| 66 |
|
---|
| 67 | public IResultParameter<IntValue> FunctionEvaluationsResultParameter {
|
---|
| 68 | get { return (IResultParameter<IntValue>)Parameters[FunctionEvaluationsResultParameterName]; }
|
---|
| 69 | }
|
---|
| 70 | public IResultParameter<IntValue> GradientEvaluationsResultParameter {
|
---|
| 71 | get { return (IResultParameter<IntValue>)Parameters[GradientEvaluationsResultParameterName]; }
|
---|
| 72 | }
|
---|
| 73 | public IFixedValueParameter<BoolValue> CountEvaluationsParameter {
|
---|
| 74 | get { return (IFixedValueParameter<BoolValue>)Parameters[CountEvaluationsParameterName]; }
|
---|
| 75 | }
|
---|
| 76 |
|
---|
| 77 |
|
---|
| 78 | public IntValue ConstantOptimizationIterations {
|
---|
| 79 | get { return ConstantOptimizationIterationsParameter.Value; }
|
---|
| 80 | }
|
---|
| 81 | public DoubleValue ConstantOptimizationImprovement {
|
---|
| 82 | get { return ConstantOptimizationImprovementParameter.Value; }
|
---|
| 83 | }
|
---|
| 84 | public PercentValue ConstantOptimizationProbability {
|
---|
| 85 | get { return ConstantOptimizationProbabilityParameter.Value; }
|
---|
| 86 | }
|
---|
| 87 | public PercentValue ConstantOptimizationRowsPercentage {
|
---|
| 88 | get { return ConstantOptimizationRowsPercentageParameter.Value; }
|
---|
| 89 | }
|
---|
| 90 | public bool UpdateConstantsInTree {
|
---|
| 91 | get { return UpdateConstantsInTreeParameter.Value.Value; }
|
---|
| 92 | set { UpdateConstantsInTreeParameter.Value.Value = value; }
|
---|
| 93 | }
|
---|
| 94 |
|
---|
| 95 | public bool UpdateVariableWeights {
|
---|
| 96 | get { return UpdateVariableWeightsParameter.Value.Value; }
|
---|
| 97 | set { UpdateVariableWeightsParameter.Value.Value = value; }
|
---|
| 98 | }
|
---|
| 99 |
|
---|
| 100 | public bool CountEvaluations {
|
---|
| 101 | get { return CountEvaluationsParameter.Value.Value; }
|
---|
| 102 | set { CountEvaluationsParameter.Value.Value = value; }
|
---|
| 103 | }
|
---|
| 104 |
|
---|
| 105 | public override bool Maximization {
|
---|
[16914] | 106 | get { return false; }
|
---|
[16912] | 107 | }
|
---|
| 108 |
|
---|
| 109 | [StorableConstructor]
|
---|
[16914] | 110 | protected ConstrainedConstantOptimizationEvaluator(StorableConstructorFlag _) : base(_) { }
|
---|
| 111 | protected ConstrainedConstantOptimizationEvaluator(ConstrainedConstantOptimizationEvaluator original, Cloner cloner)
|
---|
[16912] | 112 | : base(original, cloner) {
|
---|
| 113 | }
|
---|
[16914] | 114 | public ConstrainedConstantOptimizationEvaluator()
|
---|
[16912] | 115 | : base() {
|
---|
| 116 | Parameters.Add(new FixedValueParameter<IntValue>(ConstantOptimizationIterationsParameterName, "Determines how many iterations should be calculated while optimizing the constant of a symbolic expression tree (0 indicates other or default stopping criterion).", new IntValue(10)));
|
---|
| 117 | Parameters.Add(new FixedValueParameter<DoubleValue>(ConstantOptimizationImprovementParameterName, "Determines the relative improvement which must be achieved in the constant optimization to continue with it (0 indicates other or default stopping criterion).", new DoubleValue(0)) { Hidden = true });
|
---|
| 118 | Parameters.Add(new FixedValueParameter<PercentValue>(ConstantOptimizationProbabilityParameterName, "Determines the probability that the constants are optimized", new PercentValue(1)));
|
---|
| 119 | Parameters.Add(new FixedValueParameter<PercentValue>(ConstantOptimizationRowsPercentageParameterName, "Determines the percentage of the rows which should be used for constant optimization", new PercentValue(1)));
|
---|
| 120 | Parameters.Add(new FixedValueParameter<BoolValue>(UpdateConstantsInTreeParameterName, "Determines if the constants in the tree should be overwritten by the optimized constants.", new BoolValue(true)) { Hidden = true });
|
---|
| 121 | Parameters.Add(new FixedValueParameter<BoolValue>(UpdateVariableWeightsParameterName, "Determines if the variable weights in the tree should be optimized.", new BoolValue(true)) { Hidden = true });
|
---|
| 122 |
|
---|
| 123 | Parameters.Add(new FixedValueParameter<BoolValue>(CountEvaluationsParameterName, "Determines if function and gradient evaluation should be counted.", new BoolValue(false)));
|
---|
| 124 | Parameters.Add(new ResultParameter<IntValue>(FunctionEvaluationsResultParameterName, "The number of function evaluations performed by the constants optimization evaluator", "Results", new IntValue()));
|
---|
| 125 | Parameters.Add(new ResultParameter<IntValue>(GradientEvaluationsResultParameterName, "The number of gradient evaluations performed by the constants optimization evaluator", "Results", new IntValue()));
|
---|
| 126 | }
|
---|
| 127 |
|
---|
| 128 | public override IDeepCloneable Clone(Cloner cloner) {
|
---|
[16914] | 129 | return new ConstrainedConstantOptimizationEvaluator(this, cloner);
|
---|
[16912] | 130 | }
|
---|
| 131 |
|
---|
| 132 | [StorableHook(HookType.AfterDeserialization)]
|
---|
| 133 | private void AfterDeserialization() { }
|
---|
| 134 |
|
---|
| 135 | private static readonly object locker = new object();
|
---|
| 136 |
|
---|
| 137 | public override IOperation InstrumentedApply() {
|
---|
| 138 | var solution = SymbolicExpressionTreeParameter.ActualValue;
|
---|
| 139 | double quality;
|
---|
| 140 | if (RandomParameter.ActualValue.NextDouble() < ConstantOptimizationProbability.Value) {
|
---|
| 141 | IEnumerable<int> constantOptimizationRows = GenerateRowsToEvaluate(ConstantOptimizationRowsPercentage.Value);
|
---|
| 142 | var counter = new EvaluationsCounter();
|
---|
| 143 | quality = OptimizeConstants(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, ProblemDataParameter.ActualValue,
|
---|
| 144 | constantOptimizationRows, ApplyLinearScalingParameter.ActualValue.Value, ConstantOptimizationIterations.Value, updateVariableWeights: UpdateVariableWeights, lowerEstimationLimit: EstimationLimitsParameter.ActualValue.Lower, upperEstimationLimit: EstimationLimitsParameter.ActualValue.Upper, updateConstantsInTree: UpdateConstantsInTree, counter: counter);
|
---|
| 145 |
|
---|
| 146 | if (ConstantOptimizationRowsPercentage.Value != RelativeNumberOfEvaluatedSamplesParameter.ActualValue.Value) {
|
---|
| 147 | var evaluationRows = GenerateRowsToEvaluate();
|
---|
[16914] | 148 | quality = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, double.MinValue, double.MaxValue, ProblemDataParameter.ActualValue, evaluationRows, applyLinearScaling: false);
|
---|
[16912] | 149 | }
|
---|
| 150 |
|
---|
| 151 | if (CountEvaluations) {
|
---|
| 152 | lock (locker) {
|
---|
| 153 | FunctionEvaluationsResultParameter.ActualValue.Value += counter.FunctionEvaluations;
|
---|
| 154 | GradientEvaluationsResultParameter.ActualValue.Value += counter.GradientEvaluations;
|
---|
| 155 | }
|
---|
| 156 | }
|
---|
| 157 |
|
---|
| 158 | } else {
|
---|
| 159 | var evaluationRows = GenerateRowsToEvaluate();
|
---|
[16914] | 160 | quality = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, double.MinValue, double.MaxValue, ProblemDataParameter.ActualValue, evaluationRows, applyLinearScaling: false);
|
---|
[16912] | 161 | }
|
---|
| 162 | QualityParameter.ActualValue = new DoubleValue(quality);
|
---|
| 163 |
|
---|
| 164 | return base.InstrumentedApply();
|
---|
| 165 | }
|
---|
| 166 |
|
---|
| 167 | public override double Evaluate(IExecutionContext context, ISymbolicExpressionTree tree, IRegressionProblemData problemData, IEnumerable<int> rows) {
|
---|
| 168 | SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = context;
|
---|
| 169 | EstimationLimitsParameter.ExecutionContext = context;
|
---|
| 170 | ApplyLinearScalingParameter.ExecutionContext = context;
|
---|
| 171 | FunctionEvaluationsResultParameter.ExecutionContext = context;
|
---|
| 172 | GradientEvaluationsResultParameter.ExecutionContext = context;
|
---|
| 173 |
|
---|
[16914] | 174 | // MSE evaluator is used on purpose instead of the const-opt evaluator,
|
---|
[16912] | 175 | // because Evaluate() is used to get the quality of evolved models on
|
---|
| 176 | // different partitions of the dataset (e.g., best validation model)
|
---|
[16914] | 177 | double mse = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, double.MinValue, double.MaxValue, problemData, rows, applyLinearScaling: false);
|
---|
[16912] | 178 |
|
---|
| 179 | SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null;
|
---|
| 180 | EstimationLimitsParameter.ExecutionContext = null;
|
---|
| 181 | ApplyLinearScalingParameter.ExecutionContext = null;
|
---|
| 182 | FunctionEvaluationsResultParameter.ExecutionContext = null;
|
---|
| 183 | GradientEvaluationsResultParameter.ExecutionContext = null;
|
---|
| 184 |
|
---|
[16914] | 185 | return mse;
|
---|
[16912] | 186 | }
|
---|
| 187 |
|
---|
| 188 | public class EvaluationsCounter {
|
---|
| 189 | public int FunctionEvaluations = 0;
|
---|
| 190 | public int GradientEvaluations = 0;
|
---|
| 191 | }
|
---|
| 192 |
|
---|
| 193 | private static void GetParameterNodes(ISymbolicExpressionTree tree, out List<ISymbolicExpressionTreeNode> thetaNodes, out List<double> thetaValues) {
|
---|
| 194 | thetaNodes = new List<ISymbolicExpressionTreeNode>();
|
---|
| 195 | thetaValues = new List<double>();
|
---|
| 196 |
|
---|
| 197 | var nodes = tree.IterateNodesPrefix().ToArray();
|
---|
| 198 | for (int i = 0; i < nodes.Length; ++i) {
|
---|
| 199 | var node = nodes[i];
|
---|
| 200 | if (node is VariableTreeNode variableTreeNode) {
|
---|
| 201 | thetaValues.Add(variableTreeNode.Weight);
|
---|
| 202 | thetaNodes.Add(node);
|
---|
| 203 | } else if (node is ConstantTreeNode constantTreeNode) {
|
---|
| 204 | thetaNodes.Add(node);
|
---|
| 205 | thetaValues.Add(constantTreeNode.Value);
|
---|
| 206 | }
|
---|
| 207 | }
|
---|
| 208 | }
|
---|
| 209 |
|
---|
| 210 | public static double OptimizeConstants(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
|
---|
| 211 | ISymbolicExpressionTree tree, IRegressionProblemData problemData, IEnumerable<int> rows, bool applyLinearScaling,
|
---|
| 212 | int maxIterations, bool updateVariableWeights = true,
|
---|
| 213 | double lowerEstimationLimit = double.MinValue, double upperEstimationLimit = double.MaxValue,
|
---|
| 214 | bool updateConstantsInTree = true, Action<double[], double, object> iterationCallback = null, EvaluationsCounter counter = null) {
|
---|
| 215 |
|
---|
[16914] | 216 | if (!updateVariableWeights) throw new NotSupportedException("not updating variable weights is not supported");
|
---|
| 217 | if (!updateConstantsInTree) throw new NotSupportedException("not updating tree parameters is not supported");
|
---|
| 218 | if (applyLinearScaling) throw new NotSupportedException("linear scaling is not supported");
|
---|
[16912] | 219 |
|
---|
[16914] | 220 | // we always update constants, so we don't need to calculate initial quality
|
---|
| 221 | // double originalQuality = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
|
---|
[16912] | 222 |
|
---|
[16914] | 223 | if (counter == null) counter = new EvaluationsCounter();
|
---|
| 224 | var rowEvaluationsCounter = new EvaluationsCounter();
|
---|
[16912] | 225 |
|
---|
[16914] | 226 | var intervalConstraints = problemData.IntervalConstraints;
|
---|
| 227 | var dataIntervals = problemData.VariableRanges.VariableIntervals;
|
---|
[16912] | 228 |
|
---|
[16914] | 229 | // convert constants to variables named theta...
|
---|
| 230 | var treeForDerivation = ReplaceConstWithVar(tree, out List<string> thetaNames, out List<double> thetaValues); // copies the tree
|
---|
| 231 |
|
---|
| 232 | // create trees for relevant derivatives
|
---|
| 233 | Dictionary<string, ISymbolicExpressionTree> derivatives = new Dictionary<string, ISymbolicExpressionTree>();
|
---|
| 234 | var allThetaNodes = thetaNames.Select(_ => new List<ConstantTreeNode>()).ToArray();
|
---|
| 235 | var constraintTrees = new List<ISymbolicExpressionTree>();
|
---|
| 236 | foreach (var constraint in intervalConstraints.Constraints) {
|
---|
| 237 | if (constraint.IsDerivation) {
|
---|
| 238 | if (!problemData.AllowedInputVariables.Contains(constraint.Variable))
|
---|
| 239 | throw new ArgumentException($"Invalid constraint: the variable {constraint.Variable} does not exist in the dataset.");
|
---|
| 240 | var df = DerivativeCalculator.Derive(treeForDerivation, constraint.Variable);
|
---|
| 241 |
|
---|
| 242 | // alglib requires constraint expressions of the form c(x) <= 0
|
---|
| 243 | // -> we make two expressions, one for the lower bound and one for the upper bound
|
---|
| 244 |
|
---|
| 245 | if (constraint.Interval.UpperBound < double.PositiveInfinity) {
|
---|
| 246 | var df_smaller_upper = Subtract((ISymbolicExpressionTree)df.Clone(), CreateConstant(constraint.Interval.UpperBound));
|
---|
| 247 | // convert variables named theta back to constants
|
---|
| 248 | var df_prepared = ReplaceVarWithConst(df_smaller_upper, thetaNames, thetaValues, allThetaNodes);
|
---|
| 249 | constraintTrees.Add(df_prepared);
|
---|
| 250 | }
|
---|
| 251 | if (constraint.Interval.LowerBound > double.NegativeInfinity) {
|
---|
| 252 | var df_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)df.Clone());
|
---|
| 253 | // convert variables named theta back to constants
|
---|
| 254 | var df_prepared = ReplaceVarWithConst(df_larger_lower, thetaNames, thetaValues, allThetaNodes);
|
---|
| 255 | constraintTrees.Add(df_prepared);
|
---|
| 256 | }
|
---|
| 257 | } else {
|
---|
| 258 | if (constraint.Interval.UpperBound < double.PositiveInfinity) {
|
---|
| 259 | var f_smaller_upper = Subtract((ISymbolicExpressionTree)treeForDerivation.Clone(), CreateConstant(constraint.Interval.UpperBound));
|
---|
| 260 | // convert variables named theta back to constants
|
---|
| 261 | var df_prepared = ReplaceVarWithConst(f_smaller_upper, thetaNames, thetaValues, allThetaNodes);
|
---|
| 262 | constraintTrees.Add(df_prepared);
|
---|
| 263 | }
|
---|
| 264 | if (constraint.Interval.LowerBound > double.NegativeInfinity) {
|
---|
| 265 | var f_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)treeForDerivation.Clone());
|
---|
| 266 | // convert variables named theta back to constants
|
---|
| 267 | var df_prepared = ReplaceVarWithConst(f_larger_lower, thetaNames, thetaValues, allThetaNodes);
|
---|
| 268 | constraintTrees.Add(df_prepared);
|
---|
| 269 | }
|
---|
| 270 | }
|
---|
[16912] | 271 | }
|
---|
| 272 |
|
---|
[16914] | 273 | var preparedTree = ReplaceVarWithConst(treeForDerivation, thetaNames, thetaValues, allThetaNodes);
|
---|
[16912] | 274 |
|
---|
| 275 |
|
---|
[16914] | 276 | // local function
|
---|
| 277 | void UpdateThetaValues(double[] theta) {
|
---|
| 278 | for (int i = 0; i < theta.Length; ++i) {
|
---|
| 279 | foreach (var constNode in allThetaNodes[i]) constNode.Value = theta[i];
|
---|
| 280 | }
|
---|
| 281 | }
|
---|
[16912] | 282 |
|
---|
[16914] | 283 | // buffers for calculate_jacobian
|
---|
| 284 | var target = problemData.TargetVariableTrainingValues.ToArray();
|
---|
| 285 | var fi_eval = new double[target.Length];
|
---|
| 286 | var jac_eval = new double[target.Length, thetaValues.Count];
|
---|
[16912] | 287 |
|
---|
[16914] | 288 | // define the callback used by the alglib optimizer
|
---|
| 289 | // the x argument for this callback represents our theta
|
---|
| 290 | // local function
|
---|
| 291 | void calculate_jacobian(double[] x, double[] fi, double[,] jac, object obj) {
|
---|
| 292 | UpdateThetaValues(x);
|
---|
[16912] | 293 |
|
---|
[16914] | 294 | var autoDiffEval = new VectorAutoDiffEvaluator();
|
---|
| 295 | autoDiffEval.Evaluate(preparedTree, problemData.Dataset, problemData.TrainingIndices.ToArray(),
|
---|
| 296 | GetParameterNodes(preparedTree, allThetaNodes), fi_eval, jac_eval);
|
---|
| 297 |
|
---|
| 298 | // calc sum of squared errors and gradient
|
---|
| 299 | var sse = 0.0;
|
---|
| 300 | var g = new double[x.Length];
|
---|
| 301 | for (int i = 0; i < target.Length; i++) {
|
---|
| 302 | var res = target[i] - fi_eval[i];
|
---|
| 303 | sse += 0.5 * res * res;
|
---|
| 304 | for (int j = 0; j < g.Length; j++) {
|
---|
| 305 | g[j] -= res * jac_eval[i, j];
|
---|
| 306 | }
|
---|
| 307 | }
|
---|
| 308 |
|
---|
| 309 | fi[0] = sse / target.Length;
|
---|
| 310 | for (int j = 0; j < x.Length; j++) { jac[0, j] = g[j] / target.Length; }
|
---|
| 311 |
|
---|
| 312 | var intervalEvaluator = new IntervalEvaluator();
|
---|
| 313 | for (int i = 0; i < constraintTrees.Count; i++) {
|
---|
| 314 | var interval = intervalEvaluator.Evaluate(constraintTrees[i], dataIntervals, GetParameterNodes(constraintTrees[i], allThetaNodes),
|
---|
| 315 | out double[] lowerGradient, out double[] upperGradient);
|
---|
| 316 |
|
---|
| 317 | // we transformed this to a constraint c(x) <= 0, so only the upper bound is relevant for us
|
---|
| 318 | fi[i + 1] = interval.UpperBound;
|
---|
| 319 | for (int j = 0; j < x.Length; j++) {
|
---|
| 320 | jac[i + 1, j] = upperGradient[j];
|
---|
| 321 | }
|
---|
| 322 | }
|
---|
| 323 | }
|
---|
| 324 |
|
---|
| 325 |
|
---|
| 326 |
|
---|
| 327 | alglib.minnlcstate state;
|
---|
| 328 | alglib.minnlcreport rep;
|
---|
[16912] | 329 | try {
|
---|
[16914] | 330 | alglib.minnlccreate(thetaValues.Count, thetaValues.ToArray(), out state);
|
---|
| 331 | alglib.minnlcsetalgoslp(state); // SLP is more robust but slower
|
---|
[16915] | 332 | alglib.minnlcsetbc(state, thetaValues.Select(_ => -10000.0).ToArray(), thetaValues.Select(_ => +10000.0).ToArray());
|
---|
| 333 | alglib.minnlcsetcond(state, 1E-7, maxIterations);
|
---|
[16914] | 334 | var s = Enumerable.Repeat(1d, thetaValues.Count).ToArray(); // scale is set to unit scale
|
---|
| 335 | alglib.minnlcsetscale(state, s);
|
---|
| 336 |
|
---|
| 337 | // set non-linear constraints: 0 equality constraints, constraintTrees inequality constraints
|
---|
| 338 | alglib.minnlcsetnlc(state, 0, constraintTrees.Count);
|
---|
| 339 |
|
---|
| 340 | alglib.minnlcoptimize(state, calculate_jacobian, null, null);
|
---|
| 341 | alglib.minnlcresults(state, out double[] xOpt, out rep);
|
---|
| 342 |
|
---|
| 343 |
|
---|
| 344 | // counter.FunctionEvaluations += rep.nfev; TODO
|
---|
| 345 | counter.GradientEvaluations += rep.nfev;
|
---|
| 346 |
|
---|
| 347 | if (rep.terminationtype != -8) {
|
---|
| 348 | // update parameters in tree
|
---|
| 349 | var pIdx = 0;
|
---|
[16915] | 350 | foreach (var node in tree.IterateNodesPostfix()) {
|
---|
| 351 | if(node is ConstantTreeNode constTreeNode) {
|
---|
| 352 | constTreeNode.Value = xOpt[pIdx++];
|
---|
| 353 | } else if(node is VariableTreeNode varTreeNode) {
|
---|
| 354 | varTreeNode.Weight = xOpt[pIdx++];
|
---|
| 355 | }
|
---|
[16914] | 356 | }
|
---|
| 357 |
|
---|
| 358 | // note: we keep the optimized constants even when the tree is worse.
|
---|
| 359 | }
|
---|
| 360 |
|
---|
[16912] | 361 | } catch (ArithmeticException) {
|
---|
[16914] | 362 | // eval MSE of original tree
|
---|
| 363 | return SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
|
---|
| 364 |
|
---|
[16912] | 365 | } catch (alglib.alglibexception) {
|
---|
[16914] | 366 | // eval MSE of original tree
|
---|
| 367 | return SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
|
---|
[16912] | 368 | }
|
---|
| 369 |
|
---|
| 370 |
|
---|
[16914] | 371 | // evaluate tree with updated constants
|
---|
| 372 | return SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
|
---|
| 373 | }
|
---|
| 374 |
|
---|
| 375 | #region helper
|
---|
| 376 | private static ISymbolicExpressionTreeNode[] GetParameterNodes(ISymbolicExpressionTree tree, List<ConstantTreeNode>[] allNodes) {
|
---|
| 377 | // TODO better solution necessary
|
---|
| 378 | var treeConstNodes = tree.IterateNodesPostfix().OfType<ConstantTreeNode>().ToArray();
|
---|
| 379 | var paramNodes = new ISymbolicExpressionTreeNode[allNodes.Length];
|
---|
| 380 | for (int i = 0; i < paramNodes.Length; i++) {
|
---|
| 381 | paramNodes[i] = allNodes[i].SingleOrDefault(n => treeConstNodes.Contains(n));
|
---|
[16912] | 382 | }
|
---|
[16914] | 383 | return paramNodes;
|
---|
| 384 | }
|
---|
[16912] | 385 |
|
---|
[16914] | 386 | private static ISymbolicExpressionTree ReplaceVarWithConst(ISymbolicExpressionTree tree, List<string> thetaNames, List<double> thetaValues, List<ConstantTreeNode>[] thetaNodes) {
|
---|
| 387 | var copy = (ISymbolicExpressionTree)tree.Clone();
|
---|
| 388 | var nodes = copy.IterateNodesPostfix().ToList();
|
---|
| 389 | for (int i = 0; i < nodes.Count; i++) {
|
---|
| 390 | var n = nodes[i] as VariableTreeNode;
|
---|
| 391 | if (n != null) {
|
---|
| 392 | var thetaIdx = thetaNames.IndexOf(n.VariableName);
|
---|
| 393 | if (thetaIdx >= 0) {
|
---|
| 394 | var parent = n.Parent;
|
---|
| 395 | if (thetaNodes[thetaIdx].Any()) {
|
---|
| 396 | // HACK: REUSE CONSTANT TREE NODE IN SEVERAL TREES
|
---|
| 397 | // we use this trick to allow autodiff over thetas when thetas occurr multiple times in the tree (e.g. in derived trees)
|
---|
| 398 | var constNode = thetaNodes[thetaIdx].First();
|
---|
| 399 | var childIdx = parent.IndexOfSubtree(n);
|
---|
| 400 | parent.RemoveSubtree(childIdx);
|
---|
| 401 | parent.InsertSubtree(childIdx, constNode);
|
---|
| 402 | } else {
|
---|
| 403 | var constNode = (ConstantTreeNode)CreateConstant(thetaValues[thetaIdx]);
|
---|
| 404 | var childIdx = parent.IndexOfSubtree(n);
|
---|
| 405 | parent.RemoveSubtree(childIdx);
|
---|
| 406 | parent.InsertSubtree(childIdx, constNode);
|
---|
| 407 | thetaNodes[thetaIdx].Add(constNode);
|
---|
| 408 | }
|
---|
| 409 | }
|
---|
| 410 | }
|
---|
| 411 | }
|
---|
| 412 | return copy;
|
---|
| 413 | }
|
---|
[16912] | 414 |
|
---|
[16914] | 415 | private static ISymbolicExpressionTree ReplaceConstWithVar(ISymbolicExpressionTree tree, out List<string> thetaNames, out List<double> thetaValues) {
|
---|
| 416 | thetaNames = new List<string>();
|
---|
| 417 | thetaValues = new List<double>();
|
---|
| 418 | var copy = (ISymbolicExpressionTree)tree.Clone();
|
---|
| 419 | var nodes = copy.IterateNodesPostfix().ToList();
|
---|
| 420 |
|
---|
| 421 | int n = 1;
|
---|
| 422 | for (int i = 0; i < nodes.Count; ++i) {
|
---|
| 423 | var node = nodes[i];
|
---|
| 424 | if (node is ConstantTreeNode constantTreeNode) {
|
---|
| 425 | var thetaVar = (VariableTreeNode)new Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
|
---|
| 426 | thetaVar.Weight = 1;
|
---|
| 427 | thetaVar.VariableName = $"θ{n++}";
|
---|
| 428 |
|
---|
| 429 | thetaNames.Add(thetaVar.VariableName);
|
---|
| 430 | thetaValues.Add(constantTreeNode.Value);
|
---|
| 431 |
|
---|
| 432 | var parent = constantTreeNode.Parent;
|
---|
| 433 | if (parent != null) {
|
---|
| 434 | var index = constantTreeNode.Parent.IndexOfSubtree(constantTreeNode);
|
---|
| 435 | parent.RemoveSubtree(index);
|
---|
| 436 | parent.InsertSubtree(index, thetaVar);
|
---|
| 437 | }
|
---|
| 438 | }
|
---|
[16915] | 439 | if (node is VariableTreeNode varTreeNode) {
|
---|
| 440 | var thetaVar = (VariableTreeNode)new Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
|
---|
| 441 | thetaVar.Weight = 1;
|
---|
| 442 | thetaVar.VariableName = $"θ{n++}";
|
---|
| 443 |
|
---|
| 444 | thetaNames.Add(thetaVar.VariableName);
|
---|
| 445 | thetaValues.Add(varTreeNode.Weight);
|
---|
| 446 |
|
---|
| 447 | var parent = varTreeNode.Parent;
|
---|
| 448 | if (parent != null) {
|
---|
| 449 | var index = varTreeNode.Parent.IndexOfSubtree(varTreeNode);
|
---|
| 450 | parent.RemoveSubtree(index);
|
---|
| 451 | var prodNode = MakeNode<Multiplication>();
|
---|
| 452 | varTreeNode.Weight = 1.0;
|
---|
| 453 | prodNode.AddSubtree(varTreeNode);
|
---|
| 454 | prodNode.AddSubtree(thetaVar);
|
---|
| 455 | parent.InsertSubtree(index, prodNode);
|
---|
| 456 | }
|
---|
| 457 | }
|
---|
[16912] | 458 | }
|
---|
[16914] | 459 | return copy;
|
---|
[16912] | 460 | }
|
---|
| 461 |
|
---|
[16914] | 462 | private static ISymbolicExpressionTreeNode CreateConstant(double value) {
|
---|
| 463 | var constantNode = (ConstantTreeNode)new Constant().CreateTreeNode();
|
---|
| 464 | constantNode.Value = value;
|
---|
| 465 | return constantNode;
|
---|
| 466 | }
|
---|
| 467 |
|
---|
| 468 | private static ISymbolicExpressionTree Subtract(ISymbolicExpressionTree t, ISymbolicExpressionTreeNode b) {
|
---|
| 469 | var sub = MakeNode<Subtraction>(t.Root.GetSubtree(0).GetSubtree(0), b);
|
---|
| 470 | t.Root.GetSubtree(0).RemoveSubtree(0);
|
---|
| 471 | t.Root.GetSubtree(0).InsertSubtree(0, sub);
|
---|
| 472 | return t;
|
---|
| 473 | }
|
---|
| 474 | private static ISymbolicExpressionTree Subtract(ISymbolicExpressionTreeNode b, ISymbolicExpressionTree t) {
|
---|
| 475 | var sub = MakeNode<Subtraction>(b, t.Root.GetSubtree(0).GetSubtree(0));
|
---|
| 476 | t.Root.GetSubtree(0).RemoveSubtree(0);
|
---|
| 477 | t.Root.GetSubtree(0).InsertSubtree(0, sub);
|
---|
| 478 | return t;
|
---|
| 479 | }
|
---|
| 480 |
|
---|
| 481 | private static ISymbolicExpressionTreeNode MakeNode<T>(params ISymbolicExpressionTreeNode[] fs) where T : ISymbol, new() {
|
---|
| 482 | var node = new T().CreateTreeNode();
|
---|
| 483 | foreach (var f in fs) node.AddSubtree(f);
|
---|
| 484 | return node;
|
---|
| 485 | }
|
---|
| 486 | #endregion
|
---|
| 487 |
|
---|
[16912] | 488 | private static void UpdateConstants(ISymbolicExpressionTreeNode[] nodes, double[] constants) {
|
---|
| 489 | if (nodes.Length != constants.Length) throw new InvalidOperationException();
|
---|
[16914] | 490 | for (int i = 0; i < nodes.Length; i++) {
|
---|
[16912] | 491 | if (nodes[i] is VariableTreeNode varNode) varNode.Weight = constants[i];
|
---|
| 492 | else if (nodes[i] is ConstantTreeNode constNode) constNode.Value = constants[i];
|
---|
| 493 | }
|
---|
| 494 | }
|
---|
| 495 |
|
---|
| 496 | private static alglib.ndimensional_fvec CreateFunc(ISymbolicExpressionTree tree, VectorEvaluator eval, ISymbolicExpressionTreeNode[] parameterNodes, IDataset ds, string targetVar, int[] rows) {
|
---|
| 497 | var y = ds.GetDoubleValues(targetVar, rows).ToArray();
|
---|
| 498 | return (double[] c, double[] fi, object o) => {
|
---|
| 499 | UpdateConstants(parameterNodes, c);
|
---|
| 500 | var pred = eval.Evaluate(tree, ds, rows);
|
---|
| 501 | for (int i = 0; i < fi.Length; i++)
|
---|
| 502 | fi[i] = pred[i] - y[i];
|
---|
| 503 |
|
---|
| 504 | var counter = (EvaluationsCounter)o;
|
---|
| 505 | counter.FunctionEvaluations++;
|
---|
| 506 | };
|
---|
| 507 | }
|
---|
| 508 |
|
---|
| 509 | private static alglib.ndimensional_jac CreateJac(ISymbolicExpressionTree tree, VectorAutoDiffEvaluator eval, ISymbolicExpressionTreeNode[] parameterNodes, IDataset ds, string targetVar, int[] rows) {
|
---|
| 510 | var y = ds.GetDoubleValues(targetVar, rows).ToArray();
|
---|
| 511 | return (double[] c, double[] fi, double[,] jac, object o) => {
|
---|
| 512 | UpdateConstants(parameterNodes, c);
|
---|
| 513 | eval.Evaluate(tree, ds, rows, parameterNodes, fi, jac);
|
---|
| 514 |
|
---|
| 515 | for (int i = 0; i < fi.Length; i++)
|
---|
| 516 | fi[i] -= y[i];
|
---|
| 517 |
|
---|
| 518 | var counter = (EvaluationsCounter)o;
|
---|
| 519 | counter.GradientEvaluations++;
|
---|
| 520 | };
|
---|
| 521 | }
|
---|
| 522 | public static bool CanOptimizeConstants(ISymbolicExpressionTree tree) {
|
---|
| 523 | return TreeToAutoDiffTermConverter.IsCompatible(tree);
|
---|
| 524 | }
|
---|
| 525 | }
|
---|
| 526 | }
|
---|