Changeset 16153 for branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs

Timestamp:

09/17/18 20:17:05 (6 years ago)

Author:

gkronber

Message:

#2925 added support for multiple training episodes, added simplification of models, fixed a bug in the normalization based on target variable variance

File:

• branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs (modified) (32 diffs)

branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs

@@ -36 +36 @@
 using HeuristicLab.Problems.DataAnalysis.Symbolic;
 using HeuristicLab.Problems.Instances;
+using Variable = HeuristicLab.Problems.DataAnalysis.Symbolic.Variable;
 
 namespace HeuristicLab.Problems.DynamicalSystemsModelling {
@@ -42 +43 @@
 
     public static Vector operator +(Vector a, Vector b) {
-      if (a == Zero) return b;
-      if (b == Zero) return a;
+      if(a == Zero) return b;
+      if(b == Zero) return a;
       Debug.Assert(a.arr.Length == b.arr.Length);
       var res = new double[a.arr.Length];
-      for (int i = 0; i < res.Length; i++)
+      for(int i = 0; i < res.Length; i++)
         res[i] = a.arr[i] + b.arr[i];
       return new Vector(res);
     }
     public static Vector operator -(Vector a, Vector b) {
-      if (b == Zero) return a;
-      if (a == Zero) return -b;
+      if(b == Zero) return a;
+      if(a == Zero) return -b;
       Debug.Assert(a.arr.Length == b.arr.Length);
       var res = new double[a.arr.Length];
-      for (int i = 0; i < res.Length; i++)
+      for(int i = 0; i < res.Length; i++)
         res[i] = a.arr[i] - b.arr[i];
       return new Vector(res);
     }
     public static Vector operator -(Vector v) {
-      if (v == Zero) return Zero;
-      for (int i = 0; i < v.arr.Length; i++)
+      if(v == Zero) return Zero;
+      for(int i = 0; i < v.arr.Length; i++)
         v.arr[i] = -v.arr[i];
       return v;
@@ -67 +68 @@
 
     public static Vector operator *(double s, Vector v) {
-      if (v == Zero) return Zero;
-      if (s == 0.0) return Zero;
+      if(v == Zero) return Zero;
+      if(s == 0.0) return Zero;
       var res = new double[v.arr.Length];
-      for (int i = 0; i < res.Length; i++)
+      for(int i = 0; i < res.Length; i++)
         res[i] = s * v.arr[i];
       return new Vector(res);
@@ -78 +79 @@
     }
     public static Vector operator /(double s, Vector v) {
-      if (s == 0.0) return Zero;
-      if (v == Zero) throw new ArgumentException("Division by zero vector");
+      if(s == 0.0) return Zero;
+      if(v == Zero) throw new ArgumentException("Division by zero vector");
       var res = new double[v.arr.Length];
-      for (int i = 0; i < res.Length; i++)
+      for(int i = 0; i < res.Length; i++)
         res[i] = 1.0 / v.arr[i];
       return new Vector(res);
@@ -115 +116 @@
     private const string NumberOfLatentVariablesParameterName = "Number of latent variables";
     private const string NumericIntegrationStepsParameterName = "Steps for numeric integration";
+    private const string TrainingEpisodesParameterName = "Training episodes";
     #endregion
 
@@ -140 +142 @@
     public IFixedValueParameter<IntValue> NumericIntegrationStepsParameter {
       get { return (IFixedValueParameter<IntValue>)Parameters[NumericIntegrationStepsParameterName]; }
+    }
+    public IValueParameter<ItemList<IntRange>> TrainingEpisodesParameter {
+      get { return (IValueParameter<ItemList<IntRange>>)Parameters[TrainingEpisodesParameterName]; }
     }
     #endregion
@@ -170 +175 @@
       get { return NumericIntegrationStepsParameter.Value.Value; }
     }
-
-    #endregion                                                                                      
+    public IEnumerable<IntRange> TrainingEpisodes {
+      get { return TrainingEpisodesParameter.Value; }
+    }
+
+    #endregion
 
     public event EventHandler ProblemDataChanged;
@@ -207 +215 @@
       Parameters.Add(new FixedValueParameter<IntValue>(NumberOfLatentVariablesParameterName, "Latent variables (unobserved variables) allow us to produce expressions which are integrated up and can be used in other expressions. They are handled similarly to target variables in forward simulation / integration. The difference to target variables is that there are no data to which the calculated values of latent variables are compared. Set to a small value (0 .. 5) as necessary (default = 0)", new IntValue(0)));
       Parameters.Add(new FixedValueParameter<IntValue>(NumericIntegrationStepsParameterName, "Number of steps in the numeric integration that are taken from one row to the next (set to 1 to 100). More steps makes the algorithm slower, less steps worsens the accuracy of the numeric integration scheme.", new IntValue(10)));
+      Parameters.Add(new ValueParameter<ItemList<IntRange>>(TrainingEpisodesParameterName, "A list of ranges that should be used for training, each range represents an independent episode. This overrides the TrainingSet parameter in ProblemData.", new ItemList<IntRange>()));
 
       RegisterEventHandlers();
       InitAllParameters();
 
-      // TODO: do not clear selection of target variables when the input variables are changed
+      // TODO: do not clear selection of target variables when the input variables are changed (keep selected target variables)
       // TODO: UI hangs when selecting / deselecting input variables because the encoding is updated on each item
+
     }
 
@@ -227 +237 @@
       // collect values of all target variables
       var colIdx = 0;
-      foreach (var targetVar in targetVars) {
+      foreach(var targetVar in targetVars) {
         int rowIdx = 0;
-        foreach (var value in problemData.Dataset.GetDoubleValues(targetVar, rows)) {
+        foreach(var value in problemData.Dataset.GetDoubleValues(targetVar, rows)) {
           targetValues[rowIdx, colIdx] = value;
           rowIdx++;
@@ -238 +248 @@
       var nodeIdx = new Dictionary<ISymbolicExpressionTreeNode, int>();
 
-      foreach (var tree in trees) {
-        foreach (var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
+      foreach(var tree in trees) {
+        foreach(var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
           nodeIdx.Add(node, nodeIdx.Count);
         }
@@ -247 +257 @@
 
       double[] optTheta = new double[0];
-      if (theta.Length > 0) {
+      if(theta.Length > 0) {
         alglib.minlbfgsstate state;
         alglib.minlbfgsreport report;
@@ -253 +263 @@
         alglib.minlbfgssetcond(state, 0.0, 0.0, 0.0, MaximumParameterOptimizationIterations);
         alglib.minlbfgsoptimize(state, EvaluateObjectiveAndGradient, null,
-          new object[] { trees, targetVars, problemData, nodeIdx, targetValues, rows, NumericIntegrationSteps, latentVariables }); //TODO: create a type
+          new object[] { trees, targetVars, problemData, nodeIdx, targetValues, TrainingEpisodes.ToArray(), NumericIntegrationSteps, latentVariables }); //TODO: create a type
         alglib.minlbfgsresults(state, out optTheta, out report);
 
@@ -282 +292 @@
                           * NFEV countains number of function calculations
          */
-        if (report.terminationtype < 0) return double.MaxValue;
+        if(report.terminationtype < 0) return double.MaxValue;
       }
 
@@ -289 +299 @@
       double optQuality = double.NaN;
       EvaluateObjectiveAndGradient(optTheta, ref optQuality, grad,
-        new object[] { trees, targetVars, problemData, nodeIdx, targetValues, rows, NumericIntegrationSteps, latentVariables });
-      if (double.IsNaN(optQuality) || double.IsInfinity(optQuality)) return 10E6; // return a large value (TODO: be consistent by using NMSE)
+        new object[] { trees, targetVars, problemData, nodeIdx, targetValues, TrainingEpisodes.ToArray(), NumericIntegrationSteps, latentVariables });
+      if(double.IsNaN(optQuality) || double.IsInfinity(optQuality)) return 10E6; // return a large value (TODO: be consistent by using NMSE)
 
       individual["OptTheta"] = new DoubleArray(optTheta); // write back optimized parameters so that we can use them in the Analysis method
@@ -302 +312 @@
       var nodeIdx = (Dictionary<ISymbolicExpressionTreeNode, int>)((object[])obj)[3];
       var targetValues = (double[,])((object[])obj)[4];
-      var rows = (int[])((object[])obj)[5];
+      var episodes = (IntRange[])((object[])obj)[5];
       var numericIntegrationSteps = (int)((object[])obj)[6];
       var latentVariables = (string[])((object[])obj)[7];
@@ -312 +322 @@
         targetVariables,
         latentVariables,
-        rows,
-        nodeIdx,                // TODO: is it Ok to use rows here ?
+        episodes,
+        nodeIdx,
         x, numericIntegrationSteps).ToArray();
 
 
       // for normalized MSE = 1/variance(t) * MSE(t, pred)
-      // TODO: Perf. (by standardization of target variables before evaluation of all trees)
+      // TODO: Perf. (by standardization of target variables before evaluation of all trees)      
       var invVar = Enumerable.Range(0, targetVariables.Length)
-        .Select(c => rows.Select(row => targetValues[row, c])) // colums vectors
+        .Select(c => Enumerable.Range(0, targetValues.GetLength(0)).Select(row => targetValues[row, c])) // column vectors
         .Select(vec => vec.Variance())
         .Select(v => 1.0 / v)
@@ -331 +341 @@
       var g = Vector.Zero;
       int r = 0;
-      foreach (var y_pred in predicted) {
-        for (int c = 0; c < y_pred.Length; c++) {
+      foreach(var y_pred in predicted) {
+        for(int c = 0; c < y_pred.Length; c++) {
 
           var y_pred_f = y_pred[c].Item1;
@@ -351 +361 @@
       base.Analyze(individuals, qualities, results, random);
 
-      if (!results.ContainsKey("Prediction (training)")) {
+      if(!results.ContainsKey("Prediction (training)")) {
         results.Add(new Result("Prediction (training)", typeof(ReadOnlyItemList<DataTable>)));
       }
-      if (!results.ContainsKey("Prediction (test)")) {
+      if(!results.ContainsKey("Prediction (test)")) {
         results.Add(new Result("Prediction (test)", typeof(ReadOnlyItemList<DataTable>)));
       }
-      if (!results.ContainsKey("Models")) {
-        results.Add(new Result("Models", typeof(ReadOnlyItemList<ISymbolicExpressionTree>)));
+      if(!results.ContainsKey("Models")) {
+        results.Add(new Result("Models", typeof(VariableCollection)));
       }
 
@@ -368 +378 @@
 
 
-      foreach (var tree in trees) {
-        foreach (var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
+      foreach(var tree in trees) {
+        foreach(var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
           nodeIdx.Add(node, nodeIdx.Count);
         }
@@ -378 +388 @@
 
       var trainingList = new ItemList<DataTable>();
-      var trainingRows = ProblemData.TrainingIndices.ToArray();
       var trainingPrediction = Integrate(
        trees,  // we assume trees contain expressions for the change of each target variable over time y'(t)
@@ -385 +394 @@
        targetVars,
        latentVariables,
-       trainingRows,
+       TrainingEpisodes,
        nodeIdx,
        optTheta,
        NumericIntegrationSteps).ToArray();
 
-      for (int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
+      // only for actual target values
+      var trainingRows = TrainingEpisodes.SelectMany(e => Enumerable.Range(e.Start, e.End - e.Start));
+      for(int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
         var targetVar = targetVars[colIdx];
         var trainingDataTable = new DataTable(targetVar + " prediction (training)");
@@ -409 +420 @@
        targetVars,
        latentVariables,
-       testRows,
+       new IntRange[] { ProblemData.TestPartition },
        nodeIdx,
        optTheta,
        NumericIntegrationSteps).ToArray();
 
-      for (int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
+      for(int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
         var targetVar = targetVars[colIdx];
         var testDataTable = new DataTable(targetVar + " prediction (test)");
@@ -429 +440 @@
       #region simplification of models
       // TODO the dependency of HeuristicLab.Problems.DataAnalysis.Symbolic is not ideal
-      var modelList = new ItemList<ISymbolicExpressionTree>();
-      foreach (var tree in trees) {
-        var shownTree = (ISymbolicExpressionTree)tree.Clone();
-        var constantsNodeOrig = tree.IterateNodesPrefix().Where(IsConstantNode);
-        var constantsNodeShown = shownTree.IterateNodesPrefix().Where(IsConstantNode);
-
-        foreach (var n in constantsNodeOrig.Zip(constantsNodeShown, (original, shown) => new { original, shown })) {
-          double constantsVal = optTheta[nodeIdx[n.original]];
-
-          ConstantTreeNode replacementNode = new ConstantTreeNode(new Constant()) { Value = constantsVal };
-
-          var parentNode = n.shown.Parent;
-          int replacementIndex = parentNode.IndexOfSubtree(n.shown);
-          parentNode.RemoveSubtree(replacementIndex);
-          parentNode.InsertSubtree(replacementIndex, replacementNode);
-        }
-
-        modelList.Add(shownTree);
-      }
-      results["Models"].Value = modelList.AsReadOnly();
+      var models = new VariableCollection();    // to store target var names and original version of tree
+
+      foreach(var tup in targetVars.Zip(trees, Tuple.Create)) {
+        var targetVarName = tup.Item1;
+        var tree = tup.Item2;
+
+        // when we reference HeuristicLab.Problems.DataAnalysis.Symbolic we can translate symbols
+        int nextParIdx = 0;
+        var shownTree = new SymbolicExpressionTree(TranslateTreeNode(tree.Root, optTheta, ref nextParIdx));
+
+        // var shownTree = (SymbolicExpressionTree)tree.Clone();
+        // var constantsNodeOrig = tree.IterateNodesPrefix().Where(IsConstantNode);
+        // var constantsNodeShown = shownTree.IterateNodesPrefix().Where(IsConstantNode);
+        // 
+        // foreach (var n in constantsNodeOrig.Zip(constantsNodeShown, (original, shown) => new { original, shown })) {
+        //   double constantsVal = optTheta[nodeIdx[n.original]];
+        // 
+        //   ConstantTreeNode replacementNode = new ConstantTreeNode(new Constant()) { Value = constantsVal };
+        // 
+        //   var parentNode = n.shown.Parent;
+        //   int replacementIndex = parentNode.IndexOfSubtree(n.shown);
+        //   parentNode.RemoveSubtree(replacementIndex);
+        //   parentNode.InsertSubtree(replacementIndex, replacementNode);
+        // }
+
+        var origTreeVar = new HeuristicLab.Core.Variable(targetVarName + "(original)");
+        origTreeVar.Value = (ISymbolicExpressionTree)tree.Clone();
+        models.Add(origTreeVar);
+        var simplifiedTreeVar = new HeuristicLab.Core.Variable(targetVarName + "(simplified)");
+        simplifiedTreeVar.Value = TreeSimplifier.Simplify(shownTree);
+        models.Add(simplifiedTreeVar);
+
+      }
+      results["Models"].Value = models;
       #endregion
     }
 
+    private ISymbolicExpressionTreeNode TranslateTreeNode(ISymbolicExpressionTreeNode n, double[] parameterValues, ref int nextParIdx) {
+      ISymbolicExpressionTreeNode translatedNode = null;
+      if(n.Symbol is StartSymbol) {
+        translatedNode = new StartSymbol().CreateTreeNode();
+      } else if(n.Symbol is ProgramRootSymbol) {
+        translatedNode = new ProgramRootSymbol().CreateTreeNode();
+      } else if(n.Symbol.Name == "+") {
+        translatedNode = new Addition().CreateTreeNode();
+      } else if(n.Symbol.Name == "-") {
+        translatedNode = new Subtraction().CreateTreeNode();
+      } else if(n.Symbol.Name == "*") {
+        translatedNode = new Multiplication().CreateTreeNode();
+      } else if(n.Symbol.Name == "%") {
+        translatedNode = new Division().CreateTreeNode();
+      } else if(IsConstantNode(n)) {
+        var constNode = (ConstantTreeNode)new Constant().CreateTreeNode();
+        constNode.Value = parameterValues[nextParIdx];
+        nextParIdx++;
+        translatedNode = constNode;
+      } else {
+        // assume a variable name
+        var varName = n.Symbol.Name;
+        var varNode = (VariableTreeNode)new Variable().CreateTreeNode();
+        varNode.Weight = 1.0;
+        varNode.VariableName = varName;
+        translatedNode = varNode;
+      }
+      foreach(var child in n.Subtrees) {
+        translatedNode.AddSubtree(TranslateTreeNode(child, parameterValues, ref nextParIdx));
+      }
+      return translatedNode;
+    }
 
     #region interpretation
     private static IEnumerable<Tuple<double, Vector>[]> Integrate(
-      ISymbolicExpressionTree[] trees, IDataset dataset, string[] inputVariables, string[] targetVariables, string[] latentVariables, IEnumerable<int> rows,
+      ISymbolicExpressionTree[] trees, IDataset dataset, string[] inputVariables, string[] targetVariables, string[] latentVariables, IEnumerable<IntRange> episodes,
       Dictionary<ISymbolicExpressionTreeNode, int> nodeIdx, double[] parameterValues, int numericIntegrationSteps = 100) {
 
-      int NUM_STEPS = numericIntegrationSteps ;
+      int NUM_STEPS = numericIntegrationSteps;
       double h = 1.0 / NUM_STEPS;
 
-      // return first value as stored in the dataset
-      yield return targetVariables
-        .Select(targetVar => Tuple.Create(dataset.GetDoubleValue(targetVar, rows.First()), Vector.Zero))
-        .ToArray();
-
-      // integrate forward starting with known values for the target in t0
-
-      var variableValues = new Dictionary<string, Tuple<double, Vector>>();
-      var t0 = rows.First();
-      foreach (var varName in inputVariables) {
-        variableValues.Add(varName, Tuple.Create(dataset.GetDoubleValue(varName, t0), Vector.Zero));
-      }
-      foreach (var varName in targetVariables) {
-        variableValues.Add(varName, Tuple.Create(dataset.GetDoubleValue(varName, t0), Vector.Zero));
-      }
-      // add value entries for latent variables which are also integrated
-      foreach(var latentVar in latentVariables) {
-        variableValues.Add(latentVar, Tuple.Create(0.0, Vector.Zero)); // we don't have observations for latent variables -> assume zero as starting value
-      }
-      var calculatedVariables = targetVariables.Concat(latentVariables); // TODO: must conincide with the order of trees in the encoding
-
-      foreach (var t in rows.Skip(1)) {
-        for (int step = 0; step < NUM_STEPS; step++) {
-          var deltaValues = new Dictionary<string, Tuple<double, Vector>>();
-          foreach (var tup in trees.Zip(calculatedVariables, Tuple.Create)) {
-            var tree = tup.Item1;
-            var targetVarName = tup.Item2;
-            // skip programRoot and startSymbol
-            var res = InterpretRec(tree.Root.GetSubtree(0).GetSubtree(0), variableValues, nodeIdx, parameterValues);
-            deltaValues.Add(targetVarName, res);
+      foreach(var episode in episodes) {
+        var rows = Enumerable.Range(episode.Start, episode.End - episode.Start);
+        // return first value as stored in the dataset
+        yield return targetVariables
+          .Select(targetVar => Tuple.Create(dataset.GetDoubleValue(targetVar, rows.First()), Vector.Zero))
+          .ToArray();
+
+        // integrate forward starting with known values for the target in t0
+
+        var variableValues = new Dictionary<string, Tuple<double, Vector>>();
+        var t0 = rows.First();
+        foreach(var varName in inputVariables) {
+          variableValues.Add(varName, Tuple.Create(dataset.GetDoubleValue(varName, t0), Vector.Zero));
+        }
+        foreach(var varName in targetVariables) {
+          variableValues.Add(varName, Tuple.Create(dataset.GetDoubleValue(varName, t0), Vector.Zero));
+        }
+        // add value entries for latent variables which are also integrated
+        foreach(var latentVar in latentVariables) {
+          variableValues.Add(latentVar, Tuple.Create(0.0, Vector.Zero)); // we don't have observations for latent variables -> assume zero as starting value
+        }
+        var calculatedVariables = targetVariables.Concat(latentVariables); // TODO: must conincide with the order of trees in the encoding
+
+        foreach(var t in rows.Skip(1)) {
+          for(int step = 0; step < NUM_STEPS; step++) {
+            var deltaValues = new Dictionary<string, Tuple<double, Vector>>();
+            foreach(var tup in trees.Zip(calculatedVariables, Tuple.Create)) {
+              var tree = tup.Item1;
+              var targetVarName = tup.Item2;
+              // skip programRoot and startSymbol
+              var res = InterpretRec(tree.Root.GetSubtree(0).GetSubtree(0), variableValues, nodeIdx, parameterValues);
+              deltaValues.Add(targetVarName, res);
+            }
+
+            // update variableValues for next step
+            foreach(var kvp in deltaValues) {
+              var oldVal = variableValues[kvp.Key];
+              variableValues[kvp.Key] = Tuple.Create(
+                oldVal.Item1 + h * kvp.Value.Item1,
+                oldVal.Item2 + h * kvp.Value.Item2
+              );
+            }
           }
 
-          // update variableValues for next step
-          foreach (var kvp in deltaValues) {
-            var oldVal = variableValues[kvp.Key];
-            variableValues[kvp.Key] = Tuple.Create(
-              oldVal.Item1 + h * kvp.Value.Item1,
-              oldVal.Item2 + h * kvp.Value.Item2
-            );
+          // only return the target variables for calculation of errors
+          yield return targetVariables
+            .Select(targetVar => variableValues[targetVar])
+            .ToArray();
+
+          // update for next time step
+          foreach(var varName in inputVariables) {
+            variableValues[varName] = Tuple.Create(dataset.GetDoubleValue(varName, t), Vector.Zero);
           }
-        }
-
-        // only return the target variables for calculation of errors
-        yield return targetVariables
-          .Select(targetVar => variableValues[targetVar])
-          .ToArray();
-
-        // update for next time step
-        foreach (var varName in inputVariables) {
-          variableValues[varName] = Tuple.Create(dataset.GetDoubleValue(varName, t), Vector.Zero);
         }
       }
@@ -522 +582 @@
         ) {
 
-      switch (node.Symbol.Name) {
+      switch(node.Symbol.Name) {
         case "+": {
             var l = InterpretRec(node.GetSubtree(0), variableValues, nodeIdx, parameterValues); // TODO capture all parameters into a state type for interpretation
@@ -547 +607 @@
 
             // protected division
-            if (r.Item1.IsAlmost(0.0)) {
+            if(r.Item1.IsAlmost(0.0)) {
               return Tuple.Create(0.0, Vector.Zero);
             } else {
@@ -558 +618 @@
         default: {
             // distinguish other cases
-            if (IsConstantNode(node)) {
+            if(IsConstantNode(node)) {
               var vArr = new double[parameterValues.Length]; // backing array for vector
               vArr[nodeIdx[node]] = 1.0;
@@ -590 +650 @@
     private void RegisterEventHandlers() {
       ProblemDataParameter.ValueChanged += ProblemDataParameter_ValueChanged;
-      if (ProblemDataParameter.Value != null) ProblemDataParameter.Value.Changed += ProblemData_Changed;
+      if(ProblemDataParameter.Value != null) ProblemDataParameter.Value.Changed += ProblemData_Changed;
 
       TargetVariablesParameter.ValueChanged += TargetVariablesParameter_ValueChanged;
-      if (TargetVariablesParameter.Value != null) TargetVariablesParameter.Value.CheckedItemsChanged += CheckedTargetVariablesChanged;
+      if(TargetVariablesParameter.Value != null) TargetVariablesParameter.Value.CheckedItemsChanged += CheckedTargetVariablesChanged;
 
       FunctionSetParameter.ValueChanged += FunctionSetParameter_ValueChanged;
-      if (FunctionSetParameter.Value != null) FunctionSetParameter.Value.CheckedItemsChanged += CheckedFunctionsChanged;
+      if(FunctionSetParameter.Value != null) FunctionSetParameter.Value.CheckedItemsChanged += CheckedFunctionsChanged;
 
       MaximumLengthParameter.Value.ValueChanged += MaximumLengthChanged;
@@ -641 +701 @@
       UpdateTargetVariables();        // implicitly updates other dependent parameters
       var handler = ProblemDataChanged;
-      if (handler != null) handler(this, EventArgs.Empty);
+      if(handler != null) handler(this, EventArgs.Empty);
     }
 
@@ -674 +734 @@
       var newVariablesList = new CheckedItemCollection<StringValue>(ProblemData.Dataset.VariableNames.Select(str => new StringValue(str).AsReadOnly()).ToArray()).AsReadOnly();
       var matchingItems = newVariablesList.Where(item => currentlySelectedVariables.Contains(item.Value)).ToArray();
-      foreach (var matchingItem in matchingItems) {
+      foreach(var matchingItem in matchingItems) {
         newVariablesList.SetItemCheckedState(matchingItem, true);
       }
@@ -683 +743 @@
       var encoding = new MultiEncoding();
       var g = CreateGrammar();
-      foreach (var targetVar in TargetVariables.CheckedItems) {
+      foreach(var targetVar in TargetVariables.CheckedItems) {
         encoding = encoding.Add(new SymbolicExpressionTreeEncoding(targetVar + "_tree", g, MaximumLength, MaximumLength)); // only limit by length
       }
-      for (int i = 1; i <= NumberOfLatentVariables; i++) {
+      for(int i = 1; i <= NumberOfLatentVariables; i++) {
         encoding = encoding.Add(new SymbolicExpressionTreeEncoding("λ" + i + "_tree", g, MaximumLength, MaximumLength));
       }
@@ -704 +764 @@
       //}, 1, 1);
 
-      foreach (var variableName in ProblemData.AllowedInputVariables.Union(TargetVariables.CheckedItems.Select(i => i.Value)))
+      foreach(var variableName in ProblemData.AllowedInputVariables.Union(TargetVariables.CheckedItems.Select(i => i.Value)))
         g.AddTerminalSymbol(variableName);
 
@@ -710 +770 @@
       // we generate multiple symbols to balance the probability for selecting a numeric parameter in the generation of random trees
       var numericConstantsFactor = 2.0;
-      for (int i = 0; i < numericConstantsFactor * (ProblemData.AllowedInputVariables.Count() + TargetVariables.CheckedItems.Count()); i++) {
+      for(int i = 0; i < numericConstantsFactor * (ProblemData.AllowedInputVariables.Count() + TargetVariables.CheckedItems.Count()); i++) {
         g.AddTerminalSymbol("θ" + i); // numeric parameter for which the value is optimized using AutoDiff
       }
 
       // generate symbols for latent variables
-      for (int i = 1; i <= NumberOfLatentVariables; i++) {
+      for(int i = 1; i <= NumberOfLatentVariables; i++) {
         g.AddTerminalSymbol("λ" + i); // numeric parameter for which the value is optimized using AutoDiff
       }