Index: /branches/3087_Ceres_Integration/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/NativeInterpreter.cs
===================================================================
--- /branches/3087_Ceres_Integration/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/NativeInterpreter.cs	(revision 17853)
+++ /branches/3087_Ceres_Integration/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/NativeInterpreter.cs	(revision 17853)
@@ -0,0 +1,185 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.InteropServices;
+
+using HEAL.Attic;
+
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Parameters;
+using HeuristicLab.Problems.DataAnalysis;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
+  [StorableType("91723319-8F15-4D33-B277-40AC7C7CF9AE")]
+  [Item("NativeInterpreter", "Operator calling into native C++ code for tree interpretation.")]
+  public class NativeInterpreter : ParameterizedNamedItem, ISymbolicDataAnalysisExpressionTreeInterpreter {
+    private const string EvaluatedSolutionsParameterName = "EvaluatedSolutions";
+
+    #region parameters
+    public IFixedValueParameter<IntValue> EvaluatedSolutionsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[EvaluatedSolutionsParameterName]; }
+    }
+    #endregion
+
+    #region properties
+    public int EvaluatedSolutions {
+      get { return EvaluatedSolutionsParameter.Value.Value; }
+      set { EvaluatedSolutionsParameter.Value.Value = value; }
+    }
+    #endregion
+
+    public NativeInterpreter() {
+      Parameters.Add(new FixedValueParameter<IntValue>(EvaluatedSolutionsParameterName, "A counter for the total number of solutions the interpreter has evaluated", new IntValue(0)));
+    }
+
+    [StorableConstructor]
+    protected NativeInterpreter(StorableConstructorFlag _) : base(_) { }
+
+    protected NativeInterpreter(NativeInterpreter original, Cloner cloner) : base(original, cloner) {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new NativeInterpreter(this, cloner);
+    }
+    public static NativeInstruction[] Compile(ISymbolicExpressionTree tree, IDataset dataset, Func<ISymbolicExpressionTreeNode, byte> opCodeMapper, out List<ISymbolicExpressionTreeNode> nodes) {
+      var root = tree.Root.GetSubtree(0).GetSubtree(0);
+      return Compile(root, dataset, opCodeMapper, out nodes);
+    }
+
+    public static NativeInstruction[] Compile(ISymbolicExpressionTreeNode root, IDataset dataset, Func<ISymbolicExpressionTreeNode, byte> opCodeMapper, out List<ISymbolicExpressionTreeNode> nodes) {
+      if (cachedData == null || cachedDataset != dataset) {
+        InitCache(dataset);
+      }
+
+      nodes = root.IterateNodesPrefix().ToList(); nodes.Reverse();
+      var code = new NativeInstruction[nodes.Count];
+
+      for (int i = 0; i < nodes.Count; ++i) {
+        var node = nodes[i];
+        code[i] = new NativeInstruction { Arity = (ushort)node.SubtreeCount, OpCode = opCodeMapper(node), Length = (ushort)node.GetLength(), Optimize = true };
+
+        if (node is VariableTreeNode variable) {
+          code[i].Value = variable.Weight;
+          code[i].Data = cachedData[variable.VariableName].AddrOfPinnedObject();
+        } else if (node is ConstantTreeNode constant) {
+          code[i].Value = constant.Value;
+        }
+      }
+      return code;
+    }
+
+    private readonly object syncRoot = new object();
+
+    [ThreadStatic]
+    private static Dictionary<string, GCHandle> cachedData;
+
+    [ThreadStatic]
+    private static IDataset cachedDataset;
+
+    protected static readonly HashSet<byte> supportedOpCodes = new HashSet<byte>() {
+      (byte)OpCode.Constant,
+      (byte)OpCode.Variable,
+      (byte)OpCode.Add,
+      (byte)OpCode.Sub,
+      (byte)OpCode.Mul,
+      (byte)OpCode.Div,
+      (byte)OpCode.Exp,
+      (byte)OpCode.Log,
+      (byte)OpCode.Sin,
+      (byte)OpCode.Cos,
+      (byte)OpCode.Tan,
+      (byte)OpCode.Tanh,
+      (byte)OpCode.Power,
+      (byte)OpCode.Root,
+      (byte)OpCode.SquareRoot,
+      (byte)OpCode.Square,
+      (byte)OpCode.CubeRoot,
+      (byte)OpCode.Cube,
+      (byte)OpCode.Absolute,
+      (byte)OpCode.AnalyticQuotient
+    };
+
+    public IEnumerable<double> GetSymbolicExpressionTreeValues(ISymbolicExpressionTree tree, IDataset dataset, IEnumerable<int> rows) {
+      return GetSymbolicExpressionTreeValues(tree, dataset, rows.ToArray());
+    }
+    
+    private static void InitCache(IDataset dataset) {
+      cachedDataset = dataset;
+      // cache new data (but free old data first)
+      if (cachedData != null) {
+        foreach (var gch in cachedData.Values) {
+          gch.Free();
+        }
+      }
+      cachedData = new Dictionary<string, GCHandle>();
+      foreach (var v in dataset.DoubleVariables) {
+        var values = dataset.GetDoubleValues(v).ToArray();
+        var gch = GCHandle.Alloc(values, GCHandleType.Pinned);
+        cachedData[v] = gch;
+      }
+    }
+
+    public void ClearState() {
+      if (cachedData != null) {
+        foreach (var gch in cachedData.Values) {
+          gch.Free();
+        }
+        cachedData = null;
+      }
+      cachedDataset = null;
+      EvaluatedSolutions = 0;
+    }
+
+    public void InitializeState() {
+      ClearState();
+    }
+
+    public IEnumerable<double> GetSymbolicExpressionTreeValues(ISymbolicExpressionTree tree, IDataset dataset, int[] rows) {
+      if (!rows.Any()) return Enumerable.Empty<double>();
+
+      byte mapSupportedSymbols(ISymbolicExpressionTreeNode node) {
+        var opCode = OpCodes.MapSymbolToOpCode(node);
+        if (supportedOpCodes.Contains(opCode)) return opCode;
+        else throw new NotSupportedException($"The native interpreter does not support {node.Symbol.Name}");
+      };
+      var code = Compile(tree, dataset, mapSupportedSymbols, out List<ISymbolicExpressionTreeNode> nodes);
+
+      var result = new double[rows.Length];
+      var options = new SolverOptions { /* not using any options here */ };
+
+      var summary = new OptimizationSummary(); // also not used
+      NativeWrapper.GetValues(code, rows, result, null, options, ref summary);
+
+      // when evaluation took place without any error, we can increment the counter
+      lock (syncRoot) {
+        EvaluatedSolutions++;
+      }
+
+      return result;
+    }
+  }
+}
Index: /branches/3087_Ceres_Integration/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/ParameterOptimizer.cs
===================================================================
--- /branches/3087_Ceres_Integration/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/ParameterOptimizer.cs	(revision 17853)
+++ /branches/3087_Ceres_Integration/HeuristicLab.Problems.DataAnalysis.Symbolic/3.4/Interpreter/ParameterOptimizer.cs	(revision 17853)
@@ -0,0 +1,207 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using HEAL.Attic;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Parameters;
+
+namespace HeuristicLab.Problems.DataAnalysis.Symbolic {
+  [StorableType("A624630B-0CEB-4D06-9B26-708987A7AE8F")]
+  [Item("ParameterOptimizer", "Operator calling into native C++ code for tree interpretation.")]
+  public class ParameterOptimizer : NativeInterpreter {
+    private const string UseNonmonotonicStepsParameterName = "UseNonmonotonicSteps";
+    private const string OptimizerIterationsParameterName = "OptimizerIterations";
+
+    private const string MinimizerTypeParameterName = "MinimizerType";
+    private const string LinearSolverTypeParameterName = "LinearSolverType";
+    private const string TrustRegionStrategyTypeParameterName = "TrustRegionStrategyType";
+    private const string DogLegTypeParameterName = "DogLegType";
+    private const string LineSearchDirectionTypeParameterName = "LineSearchDirectionType";
+
+    private static readonly string[] MinimizerType = new[] { "LineSearch", "TrustRegion" };
+    private static readonly string[] LinerSolverType = new[]
+    {
+      "DenseNormalCholesky",
+      "DenseQR",
+      "SparseNormalCholesky",
+      "DenseSchur",
+      "SparseSchur",
+      "IterativeSchur",
+      "ConjugateGradients"
+    };
+    private static readonly string[] TrustRegionStrategyType = new[]
+    {
+      "LevenbergMarquardt",
+      "Dogleg"
+    };
+    private static readonly string[] DoglegType = new[]
+    {
+      "Traditional",
+      "Subspace"
+    };
+    private static readonly string[] LinearSearchDirectionType = new[]
+    {
+      "SteepestDescent",
+      "NonlinearConjugateGradient",
+      "LBFGS",
+      "BFGS"
+    };
+
+    #region parameters
+    public IFixedValueParameter<IntValue> OptimizerIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[OptimizerIterationsParameterName]; }
+    }
+    public IFixedValueParameter<BoolValue> UseNonmonotonicStepsParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[UseNonmonotonicStepsParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> MinimizerTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[MinimizerTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> LinearSolverTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[LinearSolverTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> TrustRegionStrategyTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[TrustRegionStrategyTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> DogLegTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[DogLegTypeParameterName]; }
+    }
+    public IConstrainedValueParameter<StringValue> LineSearchDirectionTypeParameter {
+      get { return (IConstrainedValueParameter<StringValue>)Parameters[LineSearchDirectionTypeParameterName]; }
+    }
+    #endregion
+
+    #region parameter properties
+    public int OptimizerIterations {
+      get { return OptimizerIterationsParameter.Value.Value; }
+      set { OptimizerIterationsParameter.Value.Value = value; }
+    }
+    public bool UseNonmonotonicSteps {
+      get { return UseNonmonotonicStepsParameter.Value.Value; }
+      set { UseNonmonotonicStepsParameter.Value.Value = value; }
+    }
+    private int Minimizer {
+      get { return Array.IndexOf(MinimizerType, MinimizerTypeParameter.Value.Value); }
+    }
+    private int LinearSolver {
+      get { return Array.IndexOf(LinerSolverType, LinearSolverTypeParameter.Value.Value); }
+    }
+    private int TrustRegionStrategy {
+      get { return Array.IndexOf(TrustRegionStrategyType, TrustRegionStrategyTypeParameter.Value.Value); }
+    }
+    private int Dogleg {
+      get { return Array.IndexOf(DoglegType, DogLegTypeParameter.Value.Value); }
+    }
+    private int LineSearchDirection {
+      get { return Array.IndexOf(LinearSearchDirectionType, LineSearchDirectionTypeParameter.Value.Value); }
+    }
+    #endregion
+
+    private static IConstrainedValueParameter<StringValue> InitializeParameter(string name, string[] validValues, string value, bool hidden = true) {
+      var parameter = new ConstrainedValueParameter<StringValue>(name, new ItemSet<StringValue>(validValues.Select(x => new StringValue(x))));
+      parameter.Value = parameter.ValidValues.Single(x => x.Value == value);
+      parameter.Hidden = hidden;
+      return parameter;
+    }
+
+    [StorableConstructor]
+    protected ParameterOptimizer(StorableConstructorFlag _) : base(_) { }
+
+    public ParameterOptimizer() {
+      var minimizerTypeParameter = InitializeParameter(MinimizerTypeParameterName, MinimizerType, "TrustRegion");
+      var linearSolverTypeParameter = InitializeParameter(LinearSolverTypeParameterName, LinerSolverType, "DenseQR");
+      var trustRegionStrategyTypeParameter = InitializeParameter(TrustRegionStrategyTypeParameterName, TrustRegionStrategyType, "LevenbergMarquardt");
+      var dogLegTypeParameter = InitializeParameter(DogLegTypeParameterName, DoglegType, "Traditional");
+      var lineSearchDirectionTypeParameter = InitializeParameter(LineSearchDirectionTypeParameterName, LinearSearchDirectionType, "SteepestDescent");
+
+      Parameters.Add(new FixedValueParameter<IntValue>(OptimizerIterationsParameterName, "The number of iterations for the nonlinear least squares optimizer.", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(UseNonmonotonicStepsParameterName, "Allow the non linear least squares optimizer to make steps in parameter space that don't necessarily decrease the error, but might improve overall convergence.", new BoolValue(false)));
+      Parameters.AddRange(new[] { minimizerTypeParameter, linearSolverTypeParameter, trustRegionStrategyTypeParameter, dogLegTypeParameter, lineSearchDirectionTypeParameter });
+    }
+
+    public ParameterOptimizer(ParameterOptimizer original, Cloner cloner) : base(original, cloner) { }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new ParameterOptimizer(this, cloner);
+    }
+
+    private static byte MapSupportedSymbols(ISymbolicExpressionTreeNode node) {
+      var opCode = OpCodes.MapSymbolToOpCode(node);
+      if (supportedOpCodes.Contains(opCode)) return opCode;
+      else throw new NotSupportedException($"The native interpreter does not support {node.Symbol.Name}");
+    }
+
+    public static Dictionary<ISymbolicExpressionTreeNode, double> OptimizeTree(ISymbolicExpressionTree tree, IDataset dataset, IEnumerable<int> rows, string targetVariable, HashSet<ISymbolicExpressionTreeNode> nodesToOptimize, SolverOptions options, ref OptimizationSummary summary) {
+      var code = Compile(tree, dataset, MapSupportedSymbols, out List<ISymbolicExpressionTreeNode> nodes);
+
+      for (int i = 0; i < code.Length; ++i) {
+        code[i].Optimize = nodesToOptimize.Contains(nodes[i]);
+      }
+
+      if (options.Iterations > 0) {
+        var target = dataset.GetDoubleValues(targetVariable, rows).ToArray();
+        var rowsArray = rows.ToArray();
+        var result = new double[rowsArray.Length];
+
+        NativeWrapper.GetValues(code, rowsArray, result, target, options, ref summary);
+      }
+      return Enumerable.Range(0, code.Length).Where(i => nodes[i] is SymbolicExpressionTreeTerminalNode).ToDictionary(i => nodes[i], i => code[i].Value);
+    }
+
+    public Dictionary<ISymbolicExpressionTreeNode, double> OptimizeTree(ISymbolicExpressionTree tree, IDataset dataset, IEnumerable<int> rows, string targetVariable, HashSet<ISymbolicExpressionTreeNode> nodesToOptimize = null) {
+      var options = new SolverOptions {
+        Iterations = OptimizerIterations,
+        Minimizer = Minimizer,
+        LinearSolver = LinearSolver,
+        TrustRegionStrategy = TrustRegionStrategy,
+        Dogleg = Dogleg,
+        LineSearchDirection = LineSearchDirection,
+        UseNonmonotonicSteps = UseNonmonotonicSteps ? 1 : 0
+      };
+
+      var summary = new OptimizationSummary();
+
+      // if no nodes are specified, use all the nodes
+      if (nodesToOptimize == null) {
+        nodesToOptimize = new HashSet<ISymbolicExpressionTreeNode>(tree.IterateNodesPrefix().Where(x => x is SymbolicExpressionTreeTerminalNode));
+      }
+
+      return OptimizeTree(tree, dataset, rows, targetVariable, nodesToOptimize, options, ref summary);
+    }
+
+    public static Dictionary<ISymbolicExpressionTreeNode, double> OptimizeTree(ISymbolicExpressionTree[] terms, IDataset dataset, IEnumerable<int> rows, string targetVariable, HashSet<ISymbolicExpressionTreeNode> nodesToOptimize, SolverOptions options, double[] coeff, ref OptimizationSummary summary) {
+      if (options.Iterations == 0) {
+        // throw exception? set iterations to 100? return empty dictionary?
+        return new Dictionary<ISymbolicExpressionTreeNode, double>();
+      }
+
+      var termIndices = new int[terms.Length];
+      var totalCodeSize = 0;
+      var totalCode = new List<NativeInstruction>();
+      var totalNodes = new List<ISymbolicExpressionTreeNode>();
+
+      // internally the native wrapper takes a single array of NativeInstructions where the indices point to the individual terms
+      for (int i = 0; i < terms.Length; ++i) {
+        var code = Compile(terms[i], dataset, MapSupportedSymbols, out List<ISymbolicExpressionTreeNode> nodes);
+        for (int j = 0; j < code.Length; ++j) {
+          code[j].Optimize = nodesToOptimize.Contains(nodes[j]);
+        }
+        totalCode.AddRange(code);
+        totalNodes.AddRange(nodes);
+
+        termIndices[i] = code.Length + totalCodeSize - 1;
+        totalCodeSize += code.Length;
+      }
+      var target = dataset.GetDoubleValues(targetVariable, rows).ToArray();
+      var rowsArray = rows.ToArray();
+      var result = new double[rowsArray.Length];
+      var codeArray = totalCode.ToArray();
+
+      NativeWrapper.GetValuesVarPro(codeArray, termIndices, rowsArray, coeff, result, target, options, ref summary);
+      return Enumerable.Range(0, totalCodeSize).Where(i => codeArray[i].Optimize).ToDictionary(i => totalNodes[i], i => codeArray[i].Value);
+    }
+  }
+}
