Index: branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs
===================================================================
--- branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs	(revision 16786)
+++ branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs	(revision 16893)
@@ -51,5 +51,6 @@
     private const string FunctionSetParameterName = "Function set";
     private const string MaximumLengthParameterName = "Size limit";
-    private const string MaximumParameterOptimizationIterationsParameterName = "Max. parameter optimization iterations";
+    private const string MaximumPretuningParameterOptimizationIterationsParameterName = "Max. pre-tuning parameter optimization iterations";
+    private const string MaximumOdeParameterOptimizationIterationsParameterName = "Max. ODE parameter optimization iterations";
     private const string NumberOfLatentVariablesParameterName = "Number of latent variables";
     private const string NumericIntegrationStepsParameterName = "Steps for numeric integration";
@@ -75,6 +76,9 @@
     }
 
-    public IFixedValueParameter<IntValue> MaximumParameterOptimizationIterationsParameter {
-      get { return (IFixedValueParameter<IntValue>)Parameters[MaximumParameterOptimizationIterationsParameterName]; }
+    public IFixedValueParameter<IntValue> MaximumPretuningParameterOptimizationIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaximumPretuningParameterOptimizationIterationsParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MaximumOdeParameterOptimizationIterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaximumOdeParameterOptimizationIterationsParameterName]; }
     }
     public IFixedValueParameter<IntValue> NumberOfLatentVariablesParameter {
@@ -92,4 +96,10 @@
     public IConstrainedValueParameter<StringValue> OdeSolverParameter {
       get { return (IConstrainedValueParameter<StringValue>)Parameters[OdeSolverParameterName]; }
+    }
+    public IFixedValueParameter<DoubleValue> PretuningErrorWeight {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters["Pretuning NMSE weight"]; }
+    }
+    public IFixedValueParameter<DoubleValue> OdeErrorWeight {
+      get { return (IFixedValueParameter<DoubleValue>)Parameters["ODE NMSE weight"]; }
     }
     #endregion
@@ -113,6 +123,9 @@
       get { return MaximumLengthParameter.Value.Value; }
     }
-    public int MaximumParameterOptimizationIterations {
-      get { return MaximumParameterOptimizationIterationsParameter.Value.Value; }
+    public int MaximumPretuningParameterOptimizationIterations {
+      get { return MaximumPretuningParameterOptimizationIterationsParameter.Value.Value; }
+    }
+    public int MaximumOdeParameterOptimizationIterations {
+      get { return MaximumOdeParameterOptimizationIterationsParameter.Value.Value; }
     }
     public int NumberOfLatentVariables {
@@ -155,4 +168,21 @@
         Parameters.Add(new FixedValueParameter<BoolValue>(OptimizeParametersForEpisodesParameterName, "Flag to select if parameters should be optimized globally or for each episode individually.", new BoolValue(false)));
       }
+      int iters = 100;
+      if (Parameters.ContainsKey("Max. parameter optimization iterations")) {
+        iters = ((IFixedValueParameter<IntValue>)Parameters["Max. parameter optimization iterations"]).Value.Value;
+      }
+      if (!Parameters.ContainsKey(MaximumPretuningParameterOptimizationIterationsParameterName)) {
+        Parameters.Add(new FixedValueParameter<IntValue>(MaximumPretuningParameterOptimizationIterationsParameterName, "The maximum number of iterations for optimization of parameters of individual equations for numerical derivatives (using L-BFGS). More iterations makes the algorithm slower, fewer iterations might prevent convergence in the optimization scheme. Default = 100", new IntValue(iters)));
+      }
+      if (!Parameters.ContainsKey(MaximumOdeParameterOptimizationIterationsParameterName)) {
+        Parameters.Add(new FixedValueParameter<IntValue>(MaximumOdeParameterOptimizationIterationsParameterName, "The maximum number of iterations for optimization of the full ODE parameters (using L-BFGS). More iterations makes the algorithm slower, fewer iterations might prevent convergence in the optimization scheme. Default = 100", new IntValue(iters)));
+      }
+
+      if (!Parameters.ContainsKey("Pretuning NMSE weight"))
+        Parameters.Add(new FixedValueParameter<DoubleValue>("Pretuning NMSE weight", "For fitness weighting", new DoubleValue(0.5)));
+      if (!Parameters.ContainsKey("ODE NMSE weight"))
+        Parameters.Add(new FixedValueParameter<DoubleValue>("ODE NMSE weight", "For fitness weighting", new DoubleValue(0.5)));
+
+
       RegisterEventHandlers();
     }
@@ -174,9 +204,12 @@
       Parameters.Add(new ValueParameter<ReadOnlyCheckedItemList<StringValue>>(FunctionSetParameterName, "The list of allowed functions", functions));
       Parameters.Add(new FixedValueParameter<IntValue>(MaximumLengthParameterName, "The maximally allowed length of each expression. Set to a small value (5 - 25). Default = 10", new IntValue(10)));
-      Parameters.Add(new FixedValueParameter<IntValue>(MaximumParameterOptimizationIterationsParameterName, "The maximum number of iterations for optimization of parameters (using L-BFGS). More iterations makes the algorithm slower, fewer iterations might prevent convergence in the optimization scheme. Default = 100", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaximumPretuningParameterOptimizationIterationsParameterName, "The maximum number of iterations for optimization of parameters of individual equations for numerical derivatives (using L-BFGS). More iterations makes the algorithm slower, fewer iterations might prevent convergence in the optimization scheme. Default = 100", new IntValue(100)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaximumOdeParameterOptimizationIterationsParameterName, "The maximum number of iterations for optimization of the full ODE parameters (using L-BFGS). More iterations makes the algorithm slower, fewer iterations might prevent convergence in the optimization scheme. Default = 100", new IntValue(100)));
       Parameters.Add(new FixedValueParameter<IntValue>(NumberOfLatentVariablesParameterName, "Latent variables (unobserved variables) allow us to produce expressions which are integrated up and can be used in other expressions. They are handled similarly to target variables in forward simulation / integration. The difference to target variables is that there are no data to which the calculated values of latent variables are compared. Set to a small value (0 .. 5) as necessary (default = 0)", new IntValue(0)));
       Parameters.Add(new FixedValueParameter<IntValue>(NumericIntegrationStepsParameterName, "Number of steps in the numeric integration that are taken from one row to the next (set to 1 to 100). More steps makes the algorithm slower, less steps worsens the accuracy of the numeric integration scheme.", new IntValue(10)));
       Parameters.Add(new ValueParameter<ItemList<IntRange>>(TrainingEpisodesParameterName, "A list of ranges that should be used for training, each range represents an independent episode. This overrides the TrainingSet parameter in ProblemData.", new ItemList<IntRange>()));
       Parameters.Add(new FixedValueParameter<BoolValue>(OptimizeParametersForEpisodesParameterName, "Flag to select if parameters should be optimized globally or for each episode individually.", new BoolValue(false)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>("Pretuning NMSE weight", "For fitness weighting", new DoubleValue(0.5)));
+      Parameters.Add(new FixedValueParameter<DoubleValue>("ODE NMSE weight", "For fitness weighting", new DoubleValue(0.5)));
 
       var solversStr = new string[] { "HeuristicLab" /* , "CVODES" */};
@@ -207,5 +240,5 @@
         foreach (var episode in TrainingEpisodes) {
           // double[] optTheta;
-          double nmse = OptimizeForEpisodes(trees, problemData, targetVars, latentVariables, random, new[] { episode }, MaximumParameterOptimizationIterations, NumericIntegrationSteps, OdeSolver);
+          double nmse = OptimizeForEpisodes(trees, problemData, targetVars, latentVariables, random, new[] { episode }, MaximumPretuningParameterOptimizationIterations, NumericIntegrationSteps, OdeSolver, MaximumOdeParameterOptimizationIterations);
           // individual["OptTheta_" + eIdx] = new DoubleArray(optTheta); // write back optimized parameters so that we can use them in the Analysis method
           eIdx++;
@@ -216,5 +249,6 @@
       } else {
         // double[] optTheta;
-        double nmse = OptimizeForEpisodes(trees, problemData, targetVars, latentVariables, random, TrainingEpisodes, MaximumParameterOptimizationIterations, NumericIntegrationSteps, OdeSolver);
+        double nmse = OptimizeForEpisodes(trees, problemData, targetVars, latentVariables, random, TrainingEpisodes, MaximumPretuningParameterOptimizationIterations, NumericIntegrationSteps, OdeSolver, MaximumOdeParameterOptimizationIterations,
+          PretuningErrorWeight.Value.Value, OdeErrorWeight.Value.Value);
         // individual["OptTheta"] = new DoubleArray(optTheta); // write back optimized parameters so that we can use them in the Analysis method
         return nmse;
@@ -229,7 +263,13 @@
       IRandom random,
       IEnumerable<IntRange> episodes,
-      int maxParameterOptIterations,
+      int maxPretuningParameterOptIterations,
       int numericIntegrationSteps,
-      string odeSolver) {
+      string odeSolver,
+      int maxOdeParameterOptIterations,
+      double pretuningErrorWeight = 0.5,
+      double odeErrorWeight = 0.5
+      ) {
+
+
 
       // extract constants from trees (without trees for latent variables)
@@ -240,5 +280,5 @@
 
       // optimize parameters by fitting f(x,y) to calculated differences dy/dt(t)
-      double nmse = PreTuneParameters(trees, problemData, targetVars, latentVariables, random, episodes, maxParameterOptIterations,
+      double nmse = pretuningErrorWeight * PreTuneParameters(trees, problemData, targetVars, latentVariables, random, episodes, maxPretuningParameterOptIterations,
         initialTheta, out double[] pretunedParameters);
 
@@ -250,5 +290,5 @@
 
       // optimize parameters using integration of f(x,y) to calculate y(t)
-      nmse = OptimizeParameters(trees, problemData, targetVars, latentVariables, episodes, maxParameterOptIterations, pretunedParameters, numericIntegrationSteps, odeSolver,
+      nmse += odeErrorWeight * OptimizeParameters(trees, problemData, targetVars, latentVariables, episodes, maxOdeParameterOptIterations, pretunedParameters, numericIntegrationSteps, odeSolver,
         out double[] optTheta);
       // var optTheta = pretunedParameters;
@@ -285,6 +325,6 @@
       var latentTrees = trees.Take(latentVariables.Length).ToArray();
 
-      {
-        // first calculate values of latent variables by integration
+      // first calculate values of latent variables by integration
+      if(latentVariables.Length > 0) {
         var inputVariables = targetVars.Concat(latentTrees.SelectMany(t => t.IterateNodesPrefix().OfType<VariableTreeNode>().Select(n => n.VariableName))).Except(latentVariables).Distinct();
         var myState = new OptimizationData(latentTrees, targetVars, inputVariables.ToArray(), problemData, null, episodes.ToArray(), 10, latentVariables, "HeuristicLab");
@@ -334,6 +374,6 @@
         var paramCount = myState.nodeValueLookup.ParameterCount;
 
-        optTheta = new double[0];
-        if (initialTheta[treeIdx].Length > 0) {
+        optTheta = initialTheta[treeIdx];
+        if (initialTheta[treeIdx].Length > 0 && maxParameterOptIterations > -1) {
           try {
             alglib.minlmstate state;
@@ -398,5 +438,5 @@
       optTheta = initialTheta;
 
-      if (initialTheta.Length > 0) {
+      if (initialTheta.Length > 0 && maxParameterOptIterations > -1) {
         var lowerBounds = Enumerable.Repeat(-1000.0, initialTheta.Length).ToArray();
         var upperBounds = Enumerable.Repeat(1000.0, initialTheta.Length).ToArray();
