Changeset 17328

Timestamp:

10/11/19 20:55:22 (5 years ago)

Author:

gkronber

Message:

#2994 made some fixes in the const-opt evaluator with constraints and added some debugging capabilities

Location:

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Extensions

Files:

• ConstrainedNLSInternal.cs (modified) (7 diffs)
• NLOptEvaluator.cs (modified) (8 diffs)

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Extensions/ConstrainedNLSInternal.cs

@@ -41 +41 @@
     private double[] bestConstraintValues;
     public double[] BestConstraintValues => bestConstraintValues;
+
+    public NLOpt.nlopt_result OptResult { get; private set; }
 
     private bool disposed = false;
@@ -84 +86 @@
     private readonly VectorEvaluator eval;
     private readonly bool invalidProblem = false;
+    private readonly double targetVariance;
 
     // end internal state
@@ -112 +115 @@
       // buffers 
       target = problemData.TargetVariableTrainingValues.ToArray();
-      var targetStDev = target.StandardDeviationPop();
-      var targetVariance = targetStDev * targetStDev;
-      var targetMean = target.Average();
+      targetVariance = target.VariancePop();
       var pred = interpreter.GetSymbolicExpressionTreeValues(expr, problemData.Dataset, trainingRows).ToArray();
 
-      bestError = targetVariance;
-
-      if (pred.Any(pi => double.IsInfinity(pi) || double.IsNaN(pi))) {
+      // all trees are linearly scaled (to improve GP performance)
+      if (pred.Any(pi => double.IsInfinity(pi) || double.IsNaN(pi)) || pred.StandardDeviationPop() == 0) {
         invalidProblem = true;
-      }
-
-      // all trees are linearly scaled (to improve GP performance)
-      #region linear scaling
-      var predStDev = pred.StandardDeviationPop();
-      if (predStDev == 0) {
-        invalidProblem = true;
-      }
-      var predMean = pred.Average();
-
-      var scalingFactor = targetStDev / predStDev;
-      var offset = targetMean - predMean * scalingFactor;
-
-      scaledTree = CopyAndScaleTree(expr, scalingFactor, offset);
-      #endregion
-
-      // convert constants to variables named theta...
-      var treeForDerivation = ReplaceAndExtractParameters(scaledTree, out List<string> thetaNames, out thetaValues); // copies the tree
-
-      // create trees for relevant derivatives
-      Dictionary<string, ISymbolicExpressionTree> derivatives = new Dictionary<string, ISymbolicExpressionTree>();
-      allThetaNodes = thetaNames.Select(_ => new List<ConstantTreeNode>()).ToArray();
-      constraintTrees = new List<ISymbolicExpressionTree>();
-      foreach (var constraint in intervalConstraints.Constraints) {
-        if (!constraint.Enabled) continue;
-        if (constraint.IsDerivation) {
-          if (!problemData.AllowedInputVariables.Contains(constraint.Variable))
-            throw new ArgumentException($"Invalid constraint: the variable {constraint.Variable} does not exist in the dataset.");
-          var df = DerivativeCalculator.Derive(treeForDerivation, constraint.Variable);
-
-          // NLOpt requires constraint expressions of the form c(x) <= 0
-          // -> we make two expressions, one for the lower bound and one for the upper bound
-
-          if (constraint.Interval.UpperBound < double.PositiveInfinity) {
-            var df_smaller_upper = Subtract((ISymbolicExpressionTree)df.Clone(), CreateConstant(constraint.Interval.UpperBound));
-            // convert variables named theta back to constants
-            var df_prepared = ReplaceVarWithConst(df_smaller_upper, thetaNames, thetaValues, allThetaNodes);
-            constraintTrees.Add(df_prepared);
-          }
-          if (constraint.Interval.LowerBound > double.NegativeInfinity) {
-            var df_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)df.Clone());
-            // convert variables named theta back to constants
-            var df_prepared = ReplaceVarWithConst(df_larger_lower, thetaNames, thetaValues, allThetaNodes);
-            constraintTrees.Add(df_prepared);
-          }
+        bestError = targetVariance;
+        bestSolution = new double[0];
+        bestConstraintValues = new double[0];
+      } else {
+        #region linear scaling
+        var cov = OnlineCovarianceCalculator.Calculate(pred, target, out OnlineCalculatorError covError);
+        var scalingFactor = cov / pred.VariancePop();
+        var offset = target.Average() - scalingFactor * pred.Average();
+
+        scaledTree = CopyAndScaleTree(expr, scalingFactor, offset);
+        #endregion
+
+        // convert constants to variables named theta...
+        var treeForDerivation = ReplaceAndExtractParameters(scaledTree, out List<string> thetaNames, out thetaValues); // copies the tree
+
+        // create trees for relevant derivatives
+        Dictionary<string, ISymbolicExpressionTree> derivatives = new Dictionary<string, ISymbolicExpressionTree>();
+        allThetaNodes = thetaNames.Select(_ => new List<ConstantTreeNode>()).ToArray();
+        constraintTrees = new List<ISymbolicExpressionTree>();
+        foreach (var constraint in intervalConstraints.Constraints) {
+          if (!constraint.Enabled) continue;
+          if (constraint.IsDerivation) {
+            if (!problemData.AllowedInputVariables.Contains(constraint.Variable))
+              throw new ArgumentException($"Invalid constraint: the variable {constraint.Variable} does not exist in the dataset.");
+            var df = DerivativeCalculator.Derive(treeForDerivation, constraint.Variable);
+
+            // NLOpt requires constraint expressions of the form c(x) <= 0
+            // -> we make two expressions, one for the lower bound and one for the upper bound
+
+            if (constraint.Interval.UpperBound < double.PositiveInfinity) {
+              var df_smaller_upper = Subtract((ISymbolicExpressionTree)df.Clone(), CreateConstant(constraint.Interval.UpperBound));
+              // convert variables named theta back to constants
+              var df_prepared = ReplaceVarWithConst(df_smaller_upper, thetaNames, thetaValues, allThetaNodes);
+              constraintTrees.Add(df_prepared);
+            }
+            if (constraint.Interval.LowerBound > double.NegativeInfinity) {
+              var df_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)df.Clone());
+              // convert variables named theta back to constants
+              var df_prepared = ReplaceVarWithConst(df_larger_lower, thetaNames, thetaValues, allThetaNodes);
+              constraintTrees.Add(df_prepared);
+            }
+          } else {
+            if (constraint.Interval.UpperBound < double.PositiveInfinity) {
+              var f_smaller_upper = Subtract((ISymbolicExpressionTree)treeForDerivation.Clone(), CreateConstant(constraint.Interval.UpperBound));
+              // convert variables named theta back to constants
+              var df_prepared = ReplaceVarWithConst(f_smaller_upper, thetaNames, thetaValues, allThetaNodes);
+              constraintTrees.Add(df_prepared);
+            }
+            if (constraint.Interval.LowerBound > double.NegativeInfinity) {
+              var f_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)treeForDerivation.Clone());
+              // convert variables named theta back to constants
+              var df_prepared = ReplaceVarWithConst(f_larger_lower, thetaNames, thetaValues, allThetaNodes);
+              constraintTrees.Add(df_prepared);
+            }
+          }
+        }
+
+        preparedTree = ReplaceVarWithConst(treeForDerivation, thetaNames, thetaValues, allThetaNodes);
+        preparedTreeParameterNodes = GetParameterNodes(preparedTree, allThetaNodes);
+
+        var dim = thetaValues.Count;
+        fi_eval = new double[target.Length]; // init buffer;
+        jac_eval = new double[target.Length, dim]; // init buffer
+
+
+        var minVal = Math.Min(-1000.0, thetaValues.Min());
+        var maxVal = Math.Max(1000.0, thetaValues.Max());
+        var lb = Enumerable.Repeat(minVal, thetaValues.Count).ToArray();
+        var up = Enumerable.Repeat(maxVal, thetaValues.Count).ToArray();
+        nlopt = NLOpt.nlopt_create(GetSolver(solver), (uint)dim);
+
+        NLOpt.nlopt_set_lower_bounds(nlopt, lb);
+        NLOpt.nlopt_set_upper_bounds(nlopt, up);
+        calculateObjectiveDelegate = new NLOpt.nlopt_func(CalculateObjective); // keep a reference to the delegate (see below)
+        NLOpt.nlopt_set_min_objective(nlopt, calculateObjectiveDelegate, IntPtr.Zero); // --> without preconditioning
+
+        //preconditionDelegate = new NLOpt.nlopt_precond(PreconditionObjective);
+        //NLOpt.nlopt_set_precond_min_objective(nlopt, calculateObjectiveDelegate, preconditionDelegate, IntPtr.Zero);
+
+
+        constraintDataPtr = new IntPtr[constraintTrees.Count];
+        calculateConstraintDelegates = new NLOpt.nlopt_func[constraintTrees.Count]; // make sure we keep a reference to the delegates (otherwise GC will free delegate objects see https://stackoverflow.com/questions/7302045/callback-delegates-being-collected#7302258)
+        for (int i = 0; i < constraintTrees.Count; i++) {
+          var constraintData = new ConstraintData() { Idx = i, Tree = constraintTrees[i], ParameterNodes = GetParameterNodes(constraintTrees[i], allThetaNodes) };
+          constraintDataPtr[i] = Marshal.AllocHGlobal(Marshal.SizeOf<ConstraintData>());
+          Marshal.StructureToPtr(constraintData, constraintDataPtr[i], fDeleteOld: false);
+          calculateConstraintDelegates[i] = new NLOpt.nlopt_func(CalculateConstraint);
+          NLOpt.nlopt_add_inequality_constraint(nlopt, calculateConstraintDelegates[i], constraintDataPtr[i], 1e-8);
+          // NLOpt.nlopt_add_precond_inequality_constraint(nlopt, calculateConstraintDelegates[i], preconditionDelegate, constraintDataPtr[i], 1e-8);
+        }
+
+
+        var x = thetaValues.ToArray();  /* initial parameters */
+                                        // calculate initial quality
+        // calculate constraints of initial solution
+        double[] constraintGrad = new double[x.Length];
+        bestConstraintValues = new double[calculateConstraintDelegates.Length];
+        for (int i = 0; i < calculateConstraintDelegates.Length; i++) {
+          bestConstraintValues[i] = calculateConstraintDelegates[i].Invoke((uint)x.Length, x, constraintGrad, constraintDataPtr[i]);
+        }
+
+        // if all constraints are OK then calculate the initial error (when there is any violation we use the variance)
+        if (bestConstraintValues.Any(c => c > 0)) {
+          bestError = targetVariance;
         } else {
-          if (constraint.Interval.UpperBound < double.PositiveInfinity) {
-            var f_smaller_upper = Subtract((ISymbolicExpressionTree)treeForDerivation.Clone(), CreateConstant(constraint.Interval.UpperBound));
-            // convert variables named theta back to constants
-            var df_prepared = ReplaceVarWithConst(f_smaller_upper, thetaNames, thetaValues, allThetaNodes);
-            constraintTrees.Add(df_prepared);
-          }
-          if (constraint.Interval.LowerBound > double.NegativeInfinity) {
-            var f_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)treeForDerivation.Clone());
-            // convert variables named theta back to constants
-            var df_prepared = ReplaceVarWithConst(f_larger_lower, thetaNames, thetaValues, allThetaNodes);
-            constraintTrees.Add(df_prepared);
-          }
-        }
-      }
-
-      preparedTree = ReplaceVarWithConst(treeForDerivation, thetaNames, thetaValues, allThetaNodes);
-      preparedTreeParameterNodes = GetParameterNodes(preparedTree, allThetaNodes);
-
-      var dim = thetaValues.Count;
-      fi_eval = new double[target.Length]; // init buffer;
-      jac_eval = new double[target.Length, dim]; // init buffer
-
-
-      var minVal = Math.Min(-1000.0, thetaValues.Min());
-      var maxVal = Math.Max(1000.0, thetaValues.Max());
-      var lb = Enumerable.Repeat(minVal, thetaValues.Count).ToArray();
-      var up = Enumerable.Repeat(maxVal, thetaValues.Count).ToArray();
-      nlopt = NLOpt.nlopt_create(GetSolver(solver), (uint)dim);
-
-      NLOpt.nlopt_set_lower_bounds(nlopt, lb);
-      NLOpt.nlopt_set_upper_bounds(nlopt, up);
-      calculateObjectiveDelegate = new NLOpt.nlopt_func(CalculateObjective); // keep a reference to the delegate (see below)
-      NLOpt.nlopt_set_min_objective(nlopt, calculateObjectiveDelegate, IntPtr.Zero); // --> without preconditioning
-
-      //preconditionDelegate = new NLOpt.nlopt_precond(PreconditionObjective);
-      //NLOpt.nlopt_set_precond_min_objective(nlopt, calculateObjectiveDelegate, preconditionDelegate, IntPtr.Zero);
-
-
-      constraintDataPtr = new IntPtr[constraintTrees.Count];
-      calculateConstraintDelegates = new NLOpt.nlopt_func[constraintTrees.Count]; // make sure we keep a reference to the delegates (otherwise GC will free delegate objects see https://stackoverflow.com/questions/7302045/callback-delegates-being-collected#7302258)
-      for (int i = 0; i < constraintTrees.Count; i++) {
-        var constraintData = new ConstraintData() { Idx = i, Tree = constraintTrees[i], ParameterNodes = GetParameterNodes(constraintTrees[i], allThetaNodes) };
-        constraintDataPtr[i] = Marshal.AllocHGlobal(Marshal.SizeOf<ConstraintData>());
-        Marshal.StructureToPtr(constraintData, constraintDataPtr[i], fDeleteOld: false);
-        calculateConstraintDelegates[i] = new NLOpt.nlopt_func(CalculateConstraint);
-        NLOpt.nlopt_add_inequality_constraint(nlopt, calculateConstraintDelegates[i], constraintDataPtr[i], 1e-8);
-        // NLOpt.nlopt_add_precond_inequality_constraint(nlopt, calculateConstraintDelegates[i], preconditionDelegate, constraintDataPtr[i], 1e-8);
-      }
-
-      NLOpt.nlopt_set_ftol_rel(nlopt, ftol_rel);
-      NLOpt.nlopt_set_ftol_abs(nlopt, ftol_abs);
-      NLOpt.nlopt_set_maxtime(nlopt, maxTime);
-      NLOpt.nlopt_set_maxeval(nlopt, maxIterations);
+          bestError = CalculateObjective((uint)dim, x, null, IntPtr.Zero);
+        }
+
+        NLOpt.nlopt_set_ftol_rel(nlopt, ftol_rel);
+        NLOpt.nlopt_set_ftol_abs(nlopt, ftol_abs);
+        NLOpt.nlopt_set_maxtime(nlopt, maxTime);
+        NLOpt.nlopt_set_maxeval(nlopt, maxIterations);
+      } // end if valid problem
     }
 
@@ -234 +248 @@
       var x = thetaValues.ToArray();  /* initial guess */
       double minf = double.MaxValue; /* minimum objective value upon return */
-      var res = NLOpt.nlopt_optimize(nlopt, x, ref minf);
-
-      if (res < 0 && res != NLOpt.nlopt_result.NLOPT_FORCED_STOP) {
+      OptResult = NLOpt.nlopt_optimize(nlopt, x, ref minf);
+
+      if (OptResult < 0 && OptResult != NLOpt.nlopt_result.NLOPT_FORCED_STOP) {
         // throw new InvalidOperationException($"NLOpt failed {res} {NLOpt.nlopt_get_errmsg(nlopt)}");
         return;
-      } else /*if ( minf <= bestError ) */{
-        bestSolution = x;
-        bestError = minf;
+      } else if (!double.IsNaN(minf) && minf < double.MaxValue) {
 
         // calculate constraints of final solution
         double[] _ = new double[x.Length];
-        bestConstraintValues = new double[calculateConstraintDelegates.Length];
+        var optimizedConstraintValues = new double[calculateConstraintDelegates.Length];
         for (int i = 0; i < calculateConstraintDelegates.Length; i++) {
-          bestConstraintValues[i] = calculateConstraintDelegates[i].Invoke((uint)x.Length, x, _, constraintDataPtr[i]);
-        }
-
-        // update parameters in tree
-        UpdateParametersInTree(scaledTree, x);
-
-        if (mode == OptimizationMode.UpdateParameters) {
-          // update original expression (when called from evaluator we want to write back optimized parameters)
-          expr.Root.GetSubtree(0).RemoveSubtree(0); // delete old tree
-          expr.Root.GetSubtree(0).InsertSubtree(0,
-            scaledTree.Root.GetSubtree(0).GetSubtree(0).GetSubtree(0).GetSubtree(0) // insert the optimized sub-tree (without scaling nodes)
-            );
-        } else if (mode == OptimizationMode.UpdateParametersAndKeepLinearScaling) {
-          expr.Root.GetSubtree(0).RemoveSubtree(0); // delete old tree
-          expr.Root.GetSubtree(0).InsertSubtree(0, scaledTree.Root.GetSubtree(0).GetSubtree(0)); // insert the optimized sub-tree (including scaling nodes)
+          optimizedConstraintValues[i] = calculateConstraintDelegates[i].Invoke((uint)x.Length, x, _, constraintDataPtr[i]);
+        }
+
+        // we accept the optimized parameters when either the error has been reduced or at least one of the violated constraints was improved (all constraints < 0 are OK anyway)
+        if (minf < bestError || bestConstraintValues.Zip(optimizedConstraintValues, Tuple.Create).Any(t => t.Item1 > 0 && t.Item1 > t.Item2)) {
+
+          bestConstraintValues = optimizedConstraintValues;
+          bestSolution = x;
+          bestError = Math.Min(minf, targetVariance);  // limit the error to variance
+
+          // update parameters in tree
+          UpdateParametersInTree(scaledTree, x);
+
+          if (mode == OptimizationMode.UpdateParameters) {
+            // update original expression (when called from evaluator we want to write back optimized parameters)
+            var newChild = (ISymbolicExpressionTreeNode)scaledTree.Root.GetSubtree(0).GetSubtree(0).GetSubtree(0).GetSubtree(0).Clone(); // the optimized sub-tree (without scaling nodes), we need to clone again to remove the parameter nodes which are used in multiple trees (and have incorrect parent relations)
+            var oldChild = expr.Root.GetSubtree(0).GetSubtree(0);
+            expr.Root.GetSubtree(0).RemoveSubtree(0); // delete old tree
+            expr.Root.GetSubtree(0).InsertSubtree(0, newChild);
+
+          } else if (mode == OptimizationMode.UpdateParametersAndKeepLinearScaling) {
+            var oldChild = expr.Root.GetSubtree(0).GetSubtree(0);
+            var newChild = (ISymbolicExpressionTreeNode)scaledTree.Root.GetSubtree(0).GetSubtree(0).Clone(); //  we need to clone again to remove the parameter nodes which are used in multiple trees(and have incorrect parent relations)
+            expr.Root.GetSubtree(0).RemoveSubtree(0); // delete old tree
+            expr.Root.GetSubtree(0).InsertSubtree(0, newChild); // insert the optimized sub-tree (including scaling nodes)
+          }
         }
       }
@@ -323 +346 @@
       }
 
-      UpdateBestSolution(sse / target.Length, curX);
+      // UpdateBestSolution(sse / target.Length, curX);
       RaiseFunctionEvaluated();
 
@@ -401 +424 @@
         out double[] lowerGradient, out double[] upperGradient);
 
-      var refInterval = refIntervalEvaluator.GetSymbolicExpressionTreeInterval(constraintData.Tree, dataIntervals);
-      if (Math.Abs(interval.LowerBound - refInterval.LowerBound) > Math.Abs(interval.LowerBound) * 1e-4) throw new InvalidProgramException($"Intervals don't match. {interval.LowerBound} <> {refInterval.LowerBound}");
-      if (Math.Abs(interval.UpperBound - refInterval.UpperBound) > Math.Abs(interval.UpperBound) * 1e-4) throw new InvalidProgramException($"Intervals don't match. {interval.UpperBound} <> {refInterval.UpperBound}");
+      // compare to reference interval calculation
+      // var refInterval = refIntervalEvaluator.GetSymbolicExpressionTreeInterval(constraintData.Tree, dataIntervals);
+      // if (Math.Abs(interval.LowerBound - refInterval.LowerBound) > Math.Abs(interval.LowerBound) * 1e-4) throw new InvalidProgramException($"Intervals don't match. {interval.LowerBound} <> {refInterval.LowerBound}");
+      // if (Math.Abs(interval.UpperBound - refInterval.UpperBound) > Math.Abs(interval.UpperBound) * 1e-4) throw new InvalidProgramException($"Intervals don't match. {interval.UpperBound} <> {refInterval.UpperBound}");
 
       // we transformed this to a constraint c(x) <= 0, so only the upper bound is relevant for us
@@ -669 +693 @@
         for (int i = 0; i < constraintDataPtr.Length; i++)
           if (constraintDataPtr[i] != IntPtr.Zero) {
+            Marshal.DestroyStructure<ConstraintData>(constraintDataPtr[i]);
             Marshal.FreeHGlobal(constraintDataPtr[i]);
             constraintDataPtr[i] = IntPtr.Zero;

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Extensions/NLOptEvaluator.cs

@@ -31 +31 @@
 using HEAL.Attic;
 using System.Runtime.InteropServices;
+using System.Diagnostics;
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
@@ -47 +48 @@
     private const string CountEvaluationsParameterName = "Count Function and Gradient Evaluations";
 
+
+    private const string AchievedQualityImprovementParameterName = "AchievedQualityImprovment";
+    private const string NumberOfConstraintViolationsBeforeOptimizationParameterName = "NumberOfConstraintViolationsBeforeOptimization";
+    private const string NumberOfConstraintViolationsAfterOptimizationParameterName = "NumberOfConstraintViolationsAfterOptimization";
+    private const string ConstraintsBeforeOptimizationParameterName = "ConstraintsBeforeOptimization";
+    private const string ViolationsAfterOptimizationParameterName = "ConstraintsAfterOptimization";
+    private const string OptimizationDurationParameterName = "OptimizationDuration";
+
     public IFixedValueParameter<IntValue> ConstantOptimizationIterationsParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters[ConstantOptimizationIterationsParameterName]; }
@@ -77 +86 @@
     public IConstrainedValueParameter<StringValue> SolverParameter {
       get { return (IConstrainedValueParameter<StringValue>)Parameters["Solver"]; }
+    }
+
+
+    public ILookupParameter<DoubleValue> AchievedQualityImprovementParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[AchievedQualityImprovementParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> NumberOfConstraintViolationsBeforeOptimizationParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[NumberOfConstraintViolationsBeforeOptimizationParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> NumberOfConstraintViolationsAfterOptimizationParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[NumberOfConstraintViolationsAfterOptimizationParameterName]; }
+    }
+    public ILookupParameter<DoubleArray> ViolationsAfterOptimizationParameter {
+      get { return (ILookupParameter<DoubleArray>)Parameters[ViolationsAfterOptimizationParameterName]; }
+    }
+    public ILookupParameter<DoubleArray> ViolationsBeforeOptimizationParameter {
+      get { return (ILookupParameter<DoubleArray>)Parameters[ConstraintsBeforeOptimizationParameterName]; }
+    }
+    public ILookupParameter<DoubleValue> OptimizationDurationParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters[OptimizationDurationParameterName]; }
     }
 
@@ -134 +163 @@
       Parameters.Add(new ResultParameter<IntValue>(FunctionEvaluationsResultParameterName, "The number of function evaluations performed by the constants optimization evaluator", "Results", new IntValue()));
       Parameters.Add(new ResultParameter<IntValue>(GradientEvaluationsResultParameterName, "The number of gradient evaluations performed by the constants optimization evaluator", "Results", new IntValue()));
+
+
+
+      Parameters.Add(new LookupParameter<DoubleValue>(AchievedQualityImprovementParameterName));
+      Parameters.Add(new LookupParameter<DoubleValue>(NumberOfConstraintViolationsBeforeOptimizationParameterName));
+      Parameters.Add(new LookupParameter<DoubleValue>(NumberOfConstraintViolationsAfterOptimizationParameterName));
+      Parameters.Add(new LookupParameter<DoubleArray>(ConstraintsBeforeOptimizationParameterName));
+      Parameters.Add(new LookupParameter<DoubleArray>(ViolationsAfterOptimizationParameterName));
+      Parameters.Add(new LookupParameter<DoubleValue>(OptimizationDurationParameterName));
     }
 
@@ -141 +179 @@
 
     [StorableHook(HookType.AfterDeserialization)]
-    private void AfterDeserialization() { }
+    private void AfterDeserialization() {
+      if (!Parameters.ContainsKey(AchievedQualityImprovementParameterName)) {
+        Parameters.Add(new LookupParameter<DoubleValue>(AchievedQualityImprovementParameterName));
+        Parameters.Add(new LookupParameter<DoubleValue>(NumberOfConstraintViolationsBeforeOptimizationParameterName));
+        Parameters.Add(new LookupParameter<DoubleValue>(NumberOfConstraintViolationsAfterOptimizationParameterName));
+      }
+      if(!Parameters.ContainsKey(ConstraintsBeforeOptimizationParameterName)) {
+        Parameters.Add(new LookupParameter<DoubleArray>(ConstraintsBeforeOptimizationParameterName));
+        Parameters.Add(new LookupParameter<DoubleArray>(ViolationsAfterOptimizationParameterName));
+        Parameters.Add(new LookupParameter<DoubleValue>(OptimizationDurationParameterName));
+
+      }
+    }
 
     private static readonly object locker = new object();
@@ -151 +201 @@
         IEnumerable<int> constantOptimizationRows = GenerateRowsToEvaluate(ConstantOptimizationRowsPercentage.Value);
         var counter = new EvaluationsCounter();
-        quality = OptimizeConstants(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, ProblemDataParameter.ActualValue,
-           constantOptimizationRows, ApplyLinearScalingParameter.ActualValue.Value, Solver, ConstantOptimizationIterations.Value, updateVariableWeights: UpdateVariableWeights, lowerEstimationLimit: EstimationLimitsParameter.ActualValue.Lower, upperEstimationLimit: EstimationLimitsParameter.ActualValue.Upper, updateConstantsInTree: UpdateConstantsInTree, counter: counter);
-
         if (ConstantOptimizationRowsPercentage.Value != RelativeNumberOfEvaluatedSamplesParameter.ActualValue.Value) {
           throw new NotSupportedException();
         }
+
+        var sw = new Stopwatch();
+        sw.Start();
+        quality = OptimizeConstants(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, ProblemDataParameter.ActualValue,
+           constantOptimizationRows, ApplyLinearScalingParameter.ActualValue.Value, Solver, 
+           out double qDiff, out double[] constraintsBefore, out double[] constraintsAfter,
+           ConstantOptimizationIterations.Value, updateVariableWeights: UpdateVariableWeights, lowerEstimationLimit: EstimationLimitsParameter.ActualValue.Lower, upperEstimationLimit: EstimationLimitsParameter.ActualValue.Upper, updateConstantsInTree: UpdateConstantsInTree, counter: counter);
+
+        AchievedQualityImprovementParameter.ActualValue = new DoubleValue(qDiff);
+        NumberOfConstraintViolationsBeforeOptimizationParameter.ActualValue = new DoubleValue(constraintsBefore.Count(cv => cv > 0));
+        NumberOfConstraintViolationsAfterOptimizationParameter.ActualValue = new DoubleValue(constraintsAfter.Count(cv => cv > 0));
+        ViolationsBeforeOptimizationParameter.ActualValue = new DoubleArray(constraintsBefore);
+        ViolationsAfterOptimizationParameter.ActualValue = new DoubleArray(constraintsAfter);
+        OptimizationDurationParameter.ActualValue = new DoubleValue(sw.Elapsed.TotalSeconds);
 
         if (CountEvaluations) {
@@ -202 +263 @@
     public static double OptimizeConstants(ISymbolicDataAnalysisExpressionTreeInterpreter interpreter,
       ISymbolicExpressionTree tree, IRegressionProblemData problemData, IEnumerable<int> rows, bool applyLinearScaling,
-      string solver,
+      string solver, out double qDiff, out double[] constraintsBefore, out double[] constraintsAfter,
       int maxIterations, bool updateVariableWeights = true,
       double lowerEstimationLimit = double.MinValue, double upperEstimationLimit = double.MaxValue,
-      bool updateConstantsInTree = true, Action<double[], double, object> iterationCallback = null, EvaluationsCounter counter = null) {
+      bool updateConstantsInTree = true, Action<double[], double, object> iterationCallback = null, EvaluationsCounter counter = null
+      ) {
 
       if (!updateVariableWeights) throw new NotSupportedException("not updating variable weights is not supported");
@@ -213 +275 @@
 
       using (var state = new ConstrainedNLSInternal(solver, tree, maxIterations, problemData, 0, 0, 0)) {
+        constraintsBefore = state.BestConstraintValues;
+        double qBefore = state.BestError;
         state.Optimize(ConstrainedNLSInternal.OptimizationMode.UpdateParameters);
-        return state.BestError;
+        constraintsAfter = state.BestConstraintValues;
+        var qOpt = state.BestError;
+        if (constraintsAfter.Any(cv => cv > 1e-8)) qOpt = qBefore;
+        qDiff = qOpt - qBefore;
+        return qOpt;
       }
     }