Changeset 16914 for branches/2994-AutoDiffForIntervals

Timestamp:

05/07/19 18:36:04 (5 years ago)

Author:

gkronber

Message:

#2994: implemented a first version of an evaluator with const opt and constraints for intervals

Location:

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Extensions

Files:

• ConstrainedConstantOptimizationEvaluator.cs (copied) (copied from branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Extensions/SymbolicRegressionNewConstantOptimizationEvaluator.cs) (8 diffs)
• HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Extensions.csproj (modified) (6 diffs)

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Extensions/ConstrainedConstantOptimizationEvaluator.cs

@@ -32 +32 @@
 
 namespace HeuristicLab.Problems.DataAnalysis.Symbolic.Regression {
-  [Item("Constant Optimization Evaluator (new)", "Calculates Pearson R² of a symbolic regression solution and optimizes the constant used.")]
-  [StorableType("1D5361E9-EF73-47D2-9211-FDD39BBC1018")]
-  public class SymbolicRegressionNewConstantOptimizationEvaluator : SymbolicRegressionSingleObjectiveEvaluator {
+  [Item("Constant Optimization Evaluator (with constraints)", "")]
+  [StorableType("A8958E06-C54A-4193-862E-8315C86EB5C1")]
+  public class ConstrainedConstantOptimizationEvaluator : SymbolicRegressionSingleObjectiveEvaluator {
     private const string ConstantOptimizationIterationsParameterName = "ConstantOptimizationIterations";
     private const string ConstantOptimizationImprovementParameterName = "ConstantOptimizationImprovement";
@@ -104 +104 @@
 
     public override bool Maximization {
-      get { return true; }
+      get { return false; }
     }
 
     [StorableConstructor]
-    protected SymbolicRegressionNewConstantOptimizationEvaluator(StorableConstructorFlag _) : base(_) { }
-    protected SymbolicRegressionNewConstantOptimizationEvaluator(SymbolicRegressionNewConstantOptimizationEvaluator original, Cloner cloner)
+    protected ConstrainedConstantOptimizationEvaluator(StorableConstructorFlag _) : base(_) { }
+    protected ConstrainedConstantOptimizationEvaluator(ConstrainedConstantOptimizationEvaluator original, Cloner cloner)
       : base(original, cloner) {
     }
-    public SymbolicRegressionNewConstantOptimizationEvaluator()
+    public ConstrainedConstantOptimizationEvaluator()
       : base() {
       Parameters.Add(new FixedValueParameter<IntValue>(ConstantOptimizationIterationsParameterName, "Determines how many iterations should be calculated while optimizing the constant of a symbolic expression tree (0 indicates other or default stopping criterion).", new IntValue(10)));
@@ -127 +127 @@
 
     public override IDeepCloneable Clone(Cloner cloner) {
-      return new SymbolicRegressionNewConstantOptimizationEvaluator(this, cloner);
+      return new ConstrainedConstantOptimizationEvaluator(this, cloner);
     }
 
@@ -146 +146 @@
         if (ConstantOptimizationRowsPercentage.Value != RelativeNumberOfEvaluatedSamplesParameter.ActualValue.Value) {
           var evaluationRows = GenerateRowsToEvaluate();
-          quality = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper, ProblemDataParameter.ActualValue, evaluationRows, ApplyLinearScalingParameter.ActualValue.Value);
+          quality = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, double.MinValue, double.MaxValue, ProblemDataParameter.ActualValue, evaluationRows, applyLinearScaling: false);
         }
 
@@ -158 +158 @@
       } else {
         var evaluationRows = GenerateRowsToEvaluate();
-        quality = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper, ProblemDataParameter.ActualValue, evaluationRows, ApplyLinearScalingParameter.ActualValue.Value);
+        quality = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, solution, double.MinValue, double.MaxValue, ProblemDataParameter.ActualValue, evaluationRows, applyLinearScaling: false);
       }
       QualityParameter.ActualValue = new DoubleValue(quality);
@@ -172 +172 @@
       GradientEvaluationsResultParameter.ExecutionContext = context;
 
-      // Pearson R² evaluator is used on purpose instead of the const-opt evaluator, 
+      // MSE evaluator is used on purpose instead of the const-opt evaluator, 
       // because Evaluate() is used to get the quality of evolved models on 
       // different partitions of the dataset (e.g., best validation model)
-      double r2 = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, EstimationLimitsParameter.ActualValue.Lower, EstimationLimitsParameter.ActualValue.Upper, problemData, rows, ApplyLinearScalingParameter.ActualValue.Value);
+      double mse = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(SymbolicDataAnalysisTreeInterpreterParameter.ActualValue, tree, double.MinValue, double.MaxValue, problemData, rows, applyLinearScaling: false);
 
       SymbolicDataAnalysisTreeInterpreterParameter.ExecutionContext = null;
@@ -183 +183 @@
       GradientEvaluationsResultParameter.ExecutionContext = null;
 
-      return r2;
+      return mse;
     }
 
@@ -214 +214 @@
       bool updateConstantsInTree = true, Action<double[], double, object> iterationCallback = null, EvaluationsCounter counter = null) {
 
-      if (!updateVariableWeights) throw new NotSupportedException();
-
-      // // numeric constants in the tree become variables for constant opt
-      // // variables in the tree become parameters (fixed values) for constant opt
-      // // for each parameter (variable in the original tree) we store the 
-      // // variable name, variable value (for factor vars) and lag as a DataForVariable object.
-      // // A dictionary is used to find parameters
-      // double[] initialConstants;
-      // var parameters = new List<TreeToAutoDiffTermConverter.DataForVariable>();
-      // 
-      // TreeToAutoDiffTermConverter.ParametricFunction func;
-      // TreeToAutoDiffTermConverter.ParametricFunctionGradient func_grad;
-      // if (!TreeToAutoDiffTermConverter.TryConvertToAutoDiff(tree, updateVariableWeights, applyLinearScaling, out parameters, out initialConstants, out func, out func_grad))
-      //   throw new NotSupportedException("Could not optimize constants of symbolic expression tree due to not supported symbols used in the tree.");
-      // if (parameters.Count == 0) return 0.0; // gkronber: constant expressions always have a R² of 0.0 
-      // var parameterEntries = parameters.ToArray(); // order of entries must be the same for x
-
-
-      GetParameterNodes(tree, out List<ISymbolicExpressionTreeNode> thetaNodes, out List<double> thetaValues);
-      var initialConstants = thetaValues.ToArray();
-
-      //extract inital constants
-      double[] c;
-      if (applyLinearScaling) {
-        c = new double[initialConstants.Length + 2];
-        c[0] = 0.0;
-        c[1] = 1.0;
-        Array.Copy(initialConstants, 0, c, 2, initialConstants.Length);
-      } else {
-        c = (double[])initialConstants.Clone();
-      }
-
-      double originalQuality = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling);
+      if (!updateVariableWeights) throw new NotSupportedException("not updating variable weights is not supported");
+      if (!updateConstantsInTree) throw new NotSupportedException("not updating tree parameters is not supported");
+      if (applyLinearScaling) throw new NotSupportedException("linear scaling is not supported");
+
+      // we always update constants, so we don't need to calculate initial quality
+      // double originalQuality = SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
 
       if (counter == null) counter = new EvaluationsCounter();
       var rowEvaluationsCounter = new EvaluationsCounter();
 
-      alglib.minlmstate state;
-      alglib.minlmreport rep;
-
-      IDataset ds = problemData.Dataset;
-      double[] y = ds.GetDoubleValues(problemData.TargetVariable, rows).ToArray();
-      int n = y.Length;
-      int k = c.Length;
-
-      var trainRows = problemData.TrainingIndices.ToArray();
-      var parameterNodes = thetaNodes.ToArray();
-      alglib.ndimensional_fvec function_cx_1_func = CreateFunc(tree, new VectorEvaluator(), parameterNodes, ds, problemData.TargetVariable, trainRows);
-      alglib.ndimensional_jac function_cx_1_jac = CreateJac(tree, new VectorAutoDiffEvaluator(), parameterNodes, ds, problemData.TargetVariable, trainRows);
-      alglib.ndimensional_rep xrep = (p, f, obj) => iterationCallback(p, f, obj);
-
+      var intervalConstraints = problemData.IntervalConstraints;
+      var dataIntervals = problemData.VariableRanges.VariableIntervals;
+
+      // convert constants to variables named theta...
+      var treeForDerivation = ReplaceConstWithVar(tree, out List<string> thetaNames, out List<double> thetaValues); // copies the tree
+
+      // create trees for relevant derivatives
+      Dictionary<string, ISymbolicExpressionTree> derivatives = new Dictionary<string, ISymbolicExpressionTree>();
+      var allThetaNodes = thetaNames.Select(_ => new List<ConstantTreeNode>()).ToArray();
+      var constraintTrees = new List<ISymbolicExpressionTree>();
+      foreach (var constraint in intervalConstraints.Constraints) {
+        if (constraint.IsDerivation) {
+          if (!problemData.AllowedInputVariables.Contains(constraint.Variable))
+            throw new ArgumentException($"Invalid constraint: the variable {constraint.Variable} does not exist in the dataset.");
+          var df = DerivativeCalculator.Derive(treeForDerivation, constraint.Variable);
+
+          // alglib requires constraint expressions of the form c(x) <= 0
+          // -> we make two expressions, one for the lower bound and one for the upper bound
+
+          if (constraint.Interval.UpperBound < double.PositiveInfinity) {
+            var df_smaller_upper = Subtract((ISymbolicExpressionTree)df.Clone(), CreateConstant(constraint.Interval.UpperBound));
+            // convert variables named theta back to constants
+            var df_prepared = ReplaceVarWithConst(df_smaller_upper, thetaNames, thetaValues, allThetaNodes);
+            constraintTrees.Add(df_prepared);
+          }
+          if (constraint.Interval.LowerBound > double.NegativeInfinity) {
+            var df_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)df.Clone());
+            // convert variables named theta back to constants
+            var df_prepared = ReplaceVarWithConst(df_larger_lower, thetaNames, thetaValues, allThetaNodes);
+            constraintTrees.Add(df_prepared);
+          }
+        } else {
+          if (constraint.Interval.UpperBound < double.PositiveInfinity) {
+            var f_smaller_upper = Subtract((ISymbolicExpressionTree)treeForDerivation.Clone(), CreateConstant(constraint.Interval.UpperBound));
+            // convert variables named theta back to constants
+            var df_prepared = ReplaceVarWithConst(f_smaller_upper, thetaNames, thetaValues, allThetaNodes);
+            constraintTrees.Add(df_prepared);
+          }
+          if (constraint.Interval.LowerBound > double.NegativeInfinity) {
+            var f_larger_lower = Subtract(CreateConstant(constraint.Interval.LowerBound), (ISymbolicExpressionTree)treeForDerivation.Clone());
+            // convert variables named theta back to constants
+            var df_prepared = ReplaceVarWithConst(f_larger_lower, thetaNames, thetaValues, allThetaNodes);
+            constraintTrees.Add(df_prepared);
+          }
+        }
+      }
+
+      var preparedTree = ReplaceVarWithConst(treeForDerivation, thetaNames, thetaValues, allThetaNodes);
+
+
+      // local function
+      void UpdateThetaValues(double[] theta) {
+        for (int i = 0; i < theta.Length; ++i) {
+          foreach (var constNode in allThetaNodes[i]) constNode.Value = theta[i];
+        }
+      }
+
+      // buffers for calculate_jacobian
+      var target = problemData.TargetVariableTrainingValues.ToArray();
+      var fi_eval = new double[target.Length];
+      var jac_eval = new double[target.Length, thetaValues.Count];
+
+      // define the callback used by the alglib optimizer
+      // the x argument for this callback represents our theta
+      // local function
+      void calculate_jacobian(double[] x, double[] fi, double[,] jac, object obj) {
+        UpdateThetaValues(x);
+
+        var autoDiffEval = new VectorAutoDiffEvaluator();
+        autoDiffEval.Evaluate(preparedTree, problemData.Dataset, problemData.TrainingIndices.ToArray(),
+          GetParameterNodes(preparedTree, allThetaNodes), fi_eval, jac_eval);
+
+        // calc sum of squared errors and gradient
+        var sse = 0.0;
+        var g = new double[x.Length];
+        for (int i = 0; i < target.Length; i++) {
+          var res = target[i] - fi_eval[i];
+          sse += 0.5 * res * res;
+          for (int j = 0; j < g.Length; j++) {
+            g[j] -= res * jac_eval[i, j];
+          }
+        }
+
+        fi[0] = sse / target.Length;
+        for (int j = 0; j < x.Length; j++) { jac[0, j] = g[j] / target.Length; }
+
+        var intervalEvaluator = new IntervalEvaluator();
+        for (int i = 0; i < constraintTrees.Count; i++) {
+          var interval = intervalEvaluator.Evaluate(constraintTrees[i], dataIntervals, GetParameterNodes(constraintTrees[i], allThetaNodes),
+            out double[] lowerGradient, out double[] upperGradient);
+
+          // we transformed this to a constraint c(x) <= 0, so only the upper bound is relevant for us
+          fi[i + 1] = interval.UpperBound;
+          for (int j = 0; j < x.Length; j++) {
+            jac[i + 1, j] = upperGradient[j];
+          }
+        }
+      }
+
+
+
+      alglib.minnlcstate state;
+      alglib.minnlcreport rep;
       try {
-        alglib.minlmcreatevj(n, c, out state);
-        alglib.minlmsetcond(state, 0.0, maxIterations);
-        alglib.minlmsetxrep(state, iterationCallback != null);
-        // alglib.minlmsetgradientcheck(state, 0.001);
-        alglib.minlmoptimize(state, function_cx_1_func, function_cx_1_jac, xrep, rowEvaluationsCounter);
-        alglib.minlmresults(state, out c, out rep);
+        alglib.minnlccreate(thetaValues.Count, thetaValues.ToArray(), out state);
+        alglib.minnlcsetalgoslp(state);        // SLP is more robust but slower
+        alglib.minnlcsetcond(state, 0, maxIterations);
+        var s = Enumerable.Repeat(1d, thetaValues.Count).ToArray();  // scale is set to unit scale
+        alglib.minnlcsetscale(state, s);
+
+        // set non-linear constraints: 0 equality constraints, constraintTrees inequality constraints
+        alglib.minnlcsetnlc(state, 0, constraintTrees.Count);
+
+        alglib.minnlcoptimize(state, calculate_jacobian, null, null);
+        alglib.minnlcresults(state, out double[] xOpt, out rep);
+
+
+        // counter.FunctionEvaluations += rep.nfev; TODO
+        counter.GradientEvaluations += rep.nfev;
+
+        if (rep.terminationtype != -8) {
+          // update parameters in tree
+          var pIdx = 0;
+          foreach (var node in tree.IterateNodesPostfix().OfType<ConstantTreeNode>()) {
+            node.Value = xOpt[pIdx++];
+          }
+
+          // note: we keep the optimized constants even when the tree is worse.
+        }
+
       } catch (ArithmeticException) {
-        return originalQuality;
+        // eval MSE of original tree
+        return SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
+
       } catch (alglib.alglibexception) {
-        return originalQuality;
-      }
-
-      counter.FunctionEvaluations += rowEvaluationsCounter.FunctionEvaluations / n;
-      counter.GradientEvaluations += rowEvaluationsCounter.GradientEvaluations / n;
-
-      //retVal == -7  => constant optimization failed due to wrong gradient
-      if (rep.terminationtype != -7) {
-        if (applyLinearScaling) {
-          var tmp = new double[c.Length - 2];
-          Array.Copy(c, 2, tmp, 0, tmp.Length);
-          UpdateConstants(parameterNodes, tmp);
-        } else UpdateConstants(parameterNodes, c);
-      }
-      var quality = SymbolicRegressionSingleObjectivePearsonRSquaredEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling);
-
-      if (!updateConstantsInTree) UpdateConstants(parameterNodes, initialConstants);
-
-      if (originalQuality - quality > 0.001 || double.IsNaN(quality)) {
-        UpdateConstants(parameterNodes, initialConstants);
-        return originalQuality;
-      }
-      return quality;
-    }
+        // eval MSE of original tree
+        return SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
+      }
+
+
+      // evaluate tree with updated constants
+      return SymbolicRegressionSingleObjectiveMeanSquaredErrorEvaluator.Calculate(interpreter, tree, lowerEstimationLimit, upperEstimationLimit, problemData, rows, applyLinearScaling: false);
+    }
+
+    #region helper
+    private static ISymbolicExpressionTreeNode[] GetParameterNodes(ISymbolicExpressionTree tree, List<ConstantTreeNode>[] allNodes) {
+      // TODO better solution necessary
+      var treeConstNodes = tree.IterateNodesPostfix().OfType<ConstantTreeNode>().ToArray();
+      var paramNodes = new ISymbolicExpressionTreeNode[allNodes.Length];
+      for (int i = 0; i < paramNodes.Length; i++) {
+        paramNodes[i] = allNodes[i].SingleOrDefault(n => treeConstNodes.Contains(n));
+      }
+      return paramNodes;
+    }
+
+    private static ISymbolicExpressionTree ReplaceVarWithConst(ISymbolicExpressionTree tree, List<string> thetaNames, List<double> thetaValues, List<ConstantTreeNode>[] thetaNodes) {
+      var copy = (ISymbolicExpressionTree)tree.Clone();
+      var nodes = copy.IterateNodesPostfix().ToList();
+      for (int i = 0; i < nodes.Count; i++) {
+        var n = nodes[i] as VariableTreeNode;
+        if (n != null) {
+          var thetaIdx = thetaNames.IndexOf(n.VariableName);
+          if (thetaIdx >= 0) {
+            var parent = n.Parent;
+            if (thetaNodes[thetaIdx].Any()) {
+              // HACK: REUSE CONSTANT TREE NODE IN SEVERAL TREES
+              // we use this trick to allow autodiff over thetas when thetas occurr multiple times in the tree (e.g. in derived trees)
+              var constNode = thetaNodes[thetaIdx].First();
+              var childIdx = parent.IndexOfSubtree(n);
+              parent.RemoveSubtree(childIdx);
+              parent.InsertSubtree(childIdx, constNode);
+            } else {
+              var constNode = (ConstantTreeNode)CreateConstant(thetaValues[thetaIdx]);
+              var childIdx = parent.IndexOfSubtree(n);
+              parent.RemoveSubtree(childIdx);
+              parent.InsertSubtree(childIdx, constNode);
+              thetaNodes[thetaIdx].Add(constNode);
+            }
+          }
+        }
+      }
+      return copy;
+    }
+
+    private static ISymbolicExpressionTree ReplaceConstWithVar(ISymbolicExpressionTree tree, out List<string> thetaNames, out List<double> thetaValues) {
+      thetaNames = new List<string>();
+      thetaValues = new List<double>();
+      var copy = (ISymbolicExpressionTree)tree.Clone();
+      var nodes = copy.IterateNodesPostfix().ToList();
+
+      int n = 1;
+      for (int i = 0; i < nodes.Count; ++i) {
+        var node = nodes[i];
+        if (node is ConstantTreeNode constantTreeNode) {
+          var thetaVar = (VariableTreeNode)new Problems.DataAnalysis.Symbolic.Variable().CreateTreeNode();
+          thetaVar.Weight = 1;
+          thetaVar.VariableName = $"θ{n++}";
+
+          thetaNames.Add(thetaVar.VariableName);
+          thetaValues.Add(constantTreeNode.Value);
+
+          var parent = constantTreeNode.Parent;
+          if (parent != null) {
+            var index = constantTreeNode.Parent.IndexOfSubtree(constantTreeNode);
+            parent.RemoveSubtree(index);
+            parent.InsertSubtree(index, thetaVar);
+          }
+        }
+      }
+      return copy;
+    }
+
+    private static ISymbolicExpressionTreeNode CreateConstant(double value) {
+      var constantNode = (ConstantTreeNode)new Constant().CreateTreeNode();
+      constantNode.Value = value;
+      return constantNode;
+    }
+
+    private static ISymbolicExpressionTree Subtract(ISymbolicExpressionTree t, ISymbolicExpressionTreeNode b) {
+      var sub = MakeNode<Subtraction>(t.Root.GetSubtree(0).GetSubtree(0), b);
+      t.Root.GetSubtree(0).RemoveSubtree(0);
+      t.Root.GetSubtree(0).InsertSubtree(0, sub);
+      return t;
+    }
+    private static ISymbolicExpressionTree Subtract(ISymbolicExpressionTreeNode b, ISymbolicExpressionTree t) {
+      var sub = MakeNode<Subtraction>(b, t.Root.GetSubtree(0).GetSubtree(0));
+      t.Root.GetSubtree(0).RemoveSubtree(0);
+      t.Root.GetSubtree(0).InsertSubtree(0, sub);
+      return t;
+    }
+
+    private static ISymbolicExpressionTreeNode MakeNode<T>(params ISymbolicExpressionTreeNode[] fs) where T : ISymbol, new() {
+      var node = new T().CreateTreeNode();
+      foreach (var f in fs) node.AddSubtree(f);
+      return node;
+    }
+    #endregion
 
     private static void UpdateConstants(ISymbolicExpressionTreeNode[] nodes, double[] constants) {
       if (nodes.Length != constants.Length) throw new InvalidOperationException();
-      for(int i = 0;i<nodes.Length;i++) {
+      for (int i = 0; i < nodes.Length; i++) {
         if (nodes[i] is VariableTreeNode varNode) varNode.Weight = constants[i];
         else if (nodes[i] is ConstantTreeNode constNode) constNode.Value = constants[i];

branches/2994-AutoDiffForIntervals/HeuristicLab.Problems.DataAnalysis.Regression.Symbolic.Extensions/HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Extensions.csproj

@@ -9 +9 @@
     <AppDesignerFolder>Properties</AppDesignerFolder>
     <RootNamespace>HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Extensions</RootNamespace>
-    <AssemblyName>HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Extensions</AssemblyName>
+    <AssemblyName>HeuristicLab.Problems.DataAnalysis.Symbolic.Regression.Extensions-0.1</AssemblyName>
     <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
@@ -18 +18 @@
     <DebugType>full</DebugType>
     <Optimize>false</Optimize>
-    <OutputPath>..\..\bin\</OutputPath>
+    <OutputPath>..\bin\</OutputPath>
     <DefineConstants>DEBUG;TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
@@ -26 +26 @@
     <DebugType>pdbonly</DebugType>
     <Optimize>true</Optimize>
-    <OutputPath>..\..\bin\</OutputPath>
+    <OutputPath>..\bin\</OutputPath>
     <DefineConstants>TRACE</DefineConstants>
     <ErrorReport>prompt</ErrorReport>
@@ -40 +40 @@
     <Reference Include="ALGLIB-3.15.0">
       <HintPath>..\bin\ALGLIB-3.15.0.dll</HintPath>
+      <Private>False</Private>
     </Reference>
     <Reference Include="HEAL.Attic">
       <HintPath>..\bin\HEAL.Attic.dll</HintPath>
+      <Private>False</Private>
     </Reference>
     <Reference Include="HeuristicLab.Problems.DataAnalysis-3.4">
       <HintPath>..\bin\HeuristicLab.Problems.DataAnalysis-3.4.dll</HintPath>
+      <Private>False</Private>
     </Reference>
     <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic-3.4">
       <HintPath>..\bin\HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.dll</HintPath>
+      <Private>False</Private>
     </Reference>
     <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4">
       <HintPath>..\bin\HeuristicLab.Problems.DataAnalysis.Symbolic.Regression-3.4.dll</HintPath>
+      <Private>False</Private>
     </Reference>
     <Reference Include="System" />
@@ -63 +68 @@
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="ConstrainedConstantOptimizationEvaluator.cs" />
+    <Compile Include="Plugin.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="SymbolicRegressionNewConstantOptimizationEvaluator.cs" />
@@ -75 +82 @@
       <Project>{958b43bc-cc5c-4fa2-8628-2b3b01d890b6}</Project>
       <Name>HeuristicLab.Collections-3.3</Name>
+      <Private>False</Private>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Common\3.3\HeuristicLab.Common-3.3.csproj">
       <Project>{a9ad58b9-3ef9-4cc1-97e5-8d909039ff5c}</Project>
       <Name>HeuristicLab.Common-3.3</Name>
+      <Private>False</Private>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Core\3.3\HeuristicLab.Core-3.3.csproj">
       <Project>{c36bd924-a541-4a00-afa8-41701378ddc5}</Project>
       <Name>HeuristicLab.Core-3.3</Name>
+      <Private>False</Private>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Data\3.3\HeuristicLab.Data-3.3.csproj">
       <Project>{bbab9df5-5ef3-4ba8-ade9-b36e82114937}</Project>
       <Name>HeuristicLab.Data-3.3</Name>
+      <Private>False</Private>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding\3.4\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.csproj">
       <Project>{06d4a186-9319-48a0-bade-a2058d462eea}</Project>
       <Name>HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4</Name>
+      <Private>False</Private>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Operators\3.3\HeuristicLab.Operators-3.3.csproj">
       <Project>{23da7ff4-d5b8-41b6-aa96-f0561d24f3ee}</Project>
       <Name>HeuristicLab.Operators-3.3</Name>
+      <Private>False</Private>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Optimization\3.3\HeuristicLab.Optimization-3.3.csproj">
       <Project>{14ab8d24-25bc-400c-a846-4627aa945192}</Project>
       <Name>HeuristicLab.Optimization-3.3</Name>
+      <Private>False</Private>
     </ProjectReference>
     <ProjectReference Include="..\HeuristicLab.Parameters\3.3\HeuristicLab.Parameters-3.3.csproj">
       <Project>{56f9106a-079f-4c61-92f6-86a84c2d84b7}</Project>
       <Name>HeuristicLab.Parameters-3.3</Name>
+      <Private>False</Private>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.PluginInfrastructure\3.3\HeuristicLab.PluginInfrastructure-3.3.csproj">
+      <Project>{94186a6a-5176-4402-ae83-886557b53cca}</Project>
+      <Name>HeuristicLab.PluginInfrastructure-3.3</Name>
     </ProjectReference>
   </ItemGroup>