Index: /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs
===================================================================
--- /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs	(revision 16214)
+++ /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs	(revision 16215)
@@ -43,24 +43,24 @@
 
     public static Vector operator +(Vector a, Vector b) {
-      if(a == Zero) return b;
-      if(b == Zero) return a;
+      if (a == Zero) return b;
+      if (b == Zero) return a;
       Debug.Assert(a.arr.Length == b.arr.Length);
       var res = new double[a.arr.Length];
-      for(int i = 0; i < res.Length; i++)
+      for (int i = 0; i < res.Length; i++)
         res[i] = a.arr[i] + b.arr[i];
       return new Vector(res);
     }
     public static Vector operator -(Vector a, Vector b) {
-      if(b == Zero) return a;
-      if(a == Zero) return -b;
+      if (b == Zero) return a;
+      if (a == Zero) return -b;
       Debug.Assert(a.arr.Length == b.arr.Length);
       var res = new double[a.arr.Length];
-      for(int i = 0; i < res.Length; i++)
+      for (int i = 0; i < res.Length; i++)
         res[i] = a.arr[i] - b.arr[i];
       return new Vector(res);
     }
     public static Vector operator -(Vector v) {
-      if(v == Zero) return Zero;
-      for(int i = 0; i < v.arr.Length; i++)
+      if (v == Zero) return Zero;
+      for (int i = 0; i < v.arr.Length; i++)
         v.arr[i] = -v.arr[i];
       return v;
@@ -68,19 +68,28 @@
 
     public static Vector operator *(double s, Vector v) {
-      if(v == Zero) return Zero;
-      if(s == 0.0) return Zero;
+      if (v == Zero) return Zero;
+      if (s == 0.0) return Zero;
       var res = new double[v.arr.Length];
-      for(int i = 0; i < res.Length; i++)
+      for (int i = 0; i < res.Length; i++)
         res[i] = s * v.arr[i];
       return new Vector(res);
     }
+
     public static Vector operator *(Vector v, double s) {
       return s * v;
     }
+    public static Vector operator *(Vector u, Vector v) {
+      if (v == Zero) return Zero;
+      if (u == Zero) return Zero;
+      var res = new double[v.arr.Length];
+      for (int i = 0; i < res.Length; i++)
+        res[i] = u.arr[i] * v.arr[i];
+      return new Vector(res);
+    }
     public static Vector operator /(double s, Vector v) {
-      if(s == 0.0) return Zero;
-      if(v == Zero) throw new ArgumentException("Division by zero vector");
+      if (s == 0.0) return Zero;
+      if (v == Zero) throw new ArgumentException("Division by zero vector");
       var res = new double[v.arr.Length];
-      for(int i = 0; i < res.Length; i++)
+      for (int i = 0; i < res.Length; i++)
         res[i] = 1.0 / v.arr[i];
       return new Vector(res);
@@ -90,4 +99,14 @@
     }
 
+    public static Vector Sin(Vector s) {
+      var res = new double[s.arr.Length];
+      for (int i = 0; i < res.Length; i++) res[i] = Math.Sin(s.arr[i]);
+      return new Vector(res);
+    }
+    public static Vector Cos(Vector s) {
+      var res = new double[s.arr.Length];
+      for (int i = 0; i < res.Length; i++) res[i] = Math.Cos(s.arr[i]);
+      return new Vector(res);
+    }
 
     private readonly double[] arr; // backing array;
@@ -200,5 +219,5 @@
     [StorableHook(HookType.AfterDeserialization)]
     private void AfterDeserialization() {
-      if(!Parameters.ContainsKey(OptimizeParametersForEpisodesParameterName)) {
+      if (!Parameters.ContainsKey(OptimizeParametersForEpisodesParameterName)) {
         Parameters.Add(new FixedValueParameter<BoolValue>(OptimizeParametersForEpisodesParameterName, "Flag to select if parameters should be optimized globally or for each episode individually.", new BoolValue(false)));
       }
@@ -232,4 +251,5 @@
       // TODO: do not clear selection of target variables when the input variables are changed (keep selected target variables)
       // TODO: UI hangs when selecting / deselecting input variables because the encoding is updated on each item
+      // TODO: use training range as default training episode
 
     }
@@ -238,9 +258,9 @@
       var trees = individual.Values.Select(v => v.Value).OfType<ISymbolicExpressionTree>().ToArray(); // extract all trees from individual
 
-      if(OptimizeParametersForEpisodes) {
-        int eIdx = 0;  
+      if (OptimizeParametersForEpisodes) {
+        int eIdx = 0;
         double totalNMSE = 0.0;
         int totalSize = 0;
-        foreach(var episode in TrainingEpisodes) {
+        foreach (var episode in TrainingEpisodes) {
           double[] optTheta;
           double nmse;
@@ -270,7 +290,7 @@
       // collect values of all target variables
       var colIdx = 0;
-      foreach(var targetVar in targetVars) {
+      foreach (var targetVar in targetVars) {
         int rowIdx = 0;
-        foreach(var value in problemData.Dataset.GetDoubleValues(targetVar, rows)) {
+        foreach (var value in problemData.Dataset.GetDoubleValues(targetVar, rows)) {
           targetValues[rowIdx, colIdx] = value;
           rowIdx++;
@@ -281,6 +301,6 @@
       var nodeIdx = new Dictionary<ISymbolicExpressionTreeNode, int>();
 
-      foreach(var tree in trees) {
-        foreach(var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
+      foreach (var tree in trees) {
+        foreach (var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
           nodeIdx.Add(node, nodeIdx.Count);
         }
@@ -290,5 +310,5 @@
 
       optTheta = new double[0];
-      if(theta.Length > 0) {
+      if (theta.Length > 0) {
         alglib.minlbfgsstate state;
         alglib.minlbfgsreport report;
@@ -325,5 +345,5 @@
                           * NFEV countains number of function calculations
          */
-        if(report.terminationtype < 0) { nmse = 10E6; return; }
+        if (report.terminationtype < 0) { nmse = 10E6; return; }
       }
 
@@ -333,5 +353,5 @@
       EvaluateObjectiveAndGradient(optTheta, ref nmse, grad,
         new object[] { trees, targetVars, problemData, nodeIdx, targetValues, episodes.ToArray(), NumericIntegrationSteps, latentVariables });
-      if(double.IsNaN(nmse) || double.IsInfinity(nmse)) { nmse = 10E6; return; } // return a large value (TODO: be consistent by using NMSE)
+      if (double.IsNaN(nmse) || double.IsInfinity(nmse)) { nmse = 10E6; return; } // return a large value (TODO: be consistent by using NMSE)
     }
 
@@ -371,6 +391,6 @@
       var g = Vector.Zero;
       int r = 0;
-      foreach(var y_pred in predicted) {
-        for(int c = 0; c < y_pred.Length; c++) {
+      foreach (var y_pred in predicted) {
+        for (int c = 0; c < y_pred.Length; c++) {
 
           var y_pred_f = y_pred[c].Item1;
@@ -391,11 +411,11 @@
       base.Analyze(individuals, qualities, results, random);
 
-      if(!results.ContainsKey("Prediction (training)")) {
+      if (!results.ContainsKey("Prediction (training)")) {
         results.Add(new Result("Prediction (training)", typeof(ReadOnlyItemList<DataTable>)));
       }
-      if(!results.ContainsKey("Prediction (test)")) {
+      if (!results.ContainsKey("Prediction (test)")) {
         results.Add(new Result("Prediction (test)", typeof(ReadOnlyItemList<DataTable>)));
       }
-      if(!results.ContainsKey("Models")) {
+      if (!results.ContainsKey("Models")) {
         results.Add(new Result("Models", typeof(VariableCollection)));
       }
@@ -406,6 +426,6 @@
       // TODO extract common functionality from Evaluate and Analyze
       var nodeIdx = new Dictionary<ISymbolicExpressionTreeNode, int>();
-      foreach(var tree in trees) {
-        foreach(var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
+      foreach (var tree in trees) {
+        foreach (var node in tree.Root.IterateNodesPrefix().Where(n => IsConstantNode(n))) {
           nodeIdx.Add(node, nodeIdx.Count);
         }
@@ -417,8 +437,8 @@
       var trainingList = new ItemList<DataTable>();
 
-      if(OptimizeParametersForEpisodes) {
+      if (OptimizeParametersForEpisodes) {
         var eIdx = 0;
         var trainingPredictions = new List<Tuple<double, Vector>[][]>();
-        foreach(var episode in TrainingEpisodes) {
+        foreach (var episode in TrainingEpisodes) {
           var episodes = new[] { episode };
           var optTheta = ((DoubleArray)bestIndividualAndQuality.Item1["OptTheta_" + eIdx]).ToArray(); // see evaluate
@@ -439,5 +459,5 @@
         // only for actual target values
         var trainingRows = TrainingEpisodes.SelectMany(e => Enumerable.Range(e.Start, e.End - e.Start));
-        for(int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
+        for (int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
           var targetVar = targetVars[colIdx];
           var trainingDataTable = new DataTable(targetVar + " prediction (training)");
@@ -453,5 +473,5 @@
         var models = new VariableCollection();
 
-        foreach(var tup in targetVars.Zip(trees, Tuple.Create)) {
+        foreach (var tup in targetVars.Zip(trees, Tuple.Create)) {
           var targetVarName = tup.Item1;
           var tree = tup.Item2;
@@ -476,5 +496,5 @@
         // only for actual target values
         var trainingRows = TrainingEpisodes.SelectMany(e => Enumerable.Range(e.Start, e.End - e.Start));
-        for(int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
+        for (int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
           var targetVar = targetVars[colIdx];
           var trainingDataTable = new DataTable(targetVar + " prediction (training)");
@@ -499,5 +519,5 @@
          NumericIntegrationSteps).ToArray();
 
-        for(int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
+        for (int colIdx = 0; colIdx < targetVars.Length; colIdx++) {
           var targetVar = targetVars[colIdx];
           var testDataTable = new DataTable(targetVar + " prediction (test)");
@@ -515,5 +535,5 @@
         var models = new VariableCollection();    // to store target var names and original version of tree
 
-        foreach(var tup in targetVars.Zip(trees, Tuple.Create)) {
+        foreach (var tup in targetVars.Zip(trees, Tuple.Create)) {
           var targetVarName = tup.Item1;
           var tree = tup.Item2;
@@ -553,17 +573,21 @@
     private ISymbolicExpressionTreeNode TranslateTreeNode(ISymbolicExpressionTreeNode n, double[] parameterValues, ref int nextParIdx) {
       ISymbolicExpressionTreeNode translatedNode = null;
-      if(n.Symbol is StartSymbol) {
+      if (n.Symbol is StartSymbol) {
         translatedNode = new StartSymbol().CreateTreeNode();
-      } else if(n.Symbol is ProgramRootSymbol) {
+      } else if (n.Symbol is ProgramRootSymbol) {
         translatedNode = new ProgramRootSymbol().CreateTreeNode();
-      } else if(n.Symbol.Name == "+") {
+      } else if (n.Symbol.Name == "+") {
         translatedNode = new Addition().CreateTreeNode();
-      } else if(n.Symbol.Name == "-") {
+      } else if (n.Symbol.Name == "-") {
         translatedNode = new Subtraction().CreateTreeNode();
-      } else if(n.Symbol.Name == "*") {
+      } else if (n.Symbol.Name == "*") {
         translatedNode = new Multiplication().CreateTreeNode();
-      } else if(n.Symbol.Name == "%") {
+      } else if (n.Symbol.Name == "%") {
         translatedNode = new Division().CreateTreeNode();
-      } else if(IsConstantNode(n)) {
+      } else if (n.Symbol.Name == "sin") {
+        translatedNode = new Sine().CreateTreeNode();
+      } else if (n.Symbol.Name == "cos") {
+        translatedNode = new Cosine().CreateTreeNode();
+      } else if (IsConstantNode(n)) {
         var constNode = (ConstantTreeNode)new Constant().CreateTreeNode();
         constNode.Value = parameterValues[nextParIdx];
@@ -578,5 +602,5 @@
         translatedNode = varNode;
       }
-      foreach(var child in n.Subtrees) {
+      foreach (var child in n.Subtrees) {
         translatedNode.AddSubtree(TranslateTreeNode(child, parameterValues, ref nextParIdx));
       }
@@ -592,5 +616,5 @@
       double h = 1.0 / NUM_STEPS;
 
-      foreach(var episode in episodes) {
+      foreach (var episode in episodes) {
         var rows = Enumerable.Range(episode.Start, episode.End - episode.Start);
         // return first value as stored in the dataset
@@ -603,20 +627,20 @@
         var variableValues = new Dictionary<string, Tuple<double, Vector>>();
         var t0 = rows.First();
-        foreach(var varName in inputVariables) {
+        foreach (var varName in inputVariables) {
           variableValues.Add(varName, Tuple.Create(dataset.GetDoubleValue(varName, t0), Vector.Zero));
         }
-        foreach(var varName in targetVariables) {
+        foreach (var varName in targetVariables) {
           variableValues.Add(varName, Tuple.Create(dataset.GetDoubleValue(varName, t0), Vector.Zero));
         }
         // add value entries for latent variables which are also integrated
-        foreach(var latentVar in latentVariables) {
+        foreach (var latentVar in latentVariables) {
           variableValues.Add(latentVar, Tuple.Create(0.0, Vector.Zero)); // we don't have observations for latent variables -> assume zero as starting value
         }
         var calculatedVariables = targetVariables.Concat(latentVariables); // TODO: must conincide with the order of trees in the encoding
 
-        foreach(var t in rows.Skip(1)) {
-          for(int step = 0; step < NUM_STEPS; step++) {
+        foreach (var t in rows.Skip(1)) {
+          for (int step = 0; step < NUM_STEPS; step++) {
             var deltaValues = new Dictionary<string, Tuple<double, Vector>>();
-            foreach(var tup in trees.Zip(calculatedVariables, Tuple.Create)) {
+            foreach (var tup in trees.Zip(calculatedVariables, Tuple.Create)) {
               var tree = tup.Item1;
               var targetVarName = tup.Item2;
@@ -627,5 +651,5 @@
 
             // update variableValues for next step
-            foreach(var kvp in deltaValues) {
+            foreach (var kvp in deltaValues) {
               var oldVal = variableValues[kvp.Key];
               variableValues[kvp.Key] = Tuple.Create(
@@ -642,5 +666,5 @@
 
           // update for next time step
-          foreach(var varName in inputVariables) {
+          foreach (var varName in inputVariables) {
             variableValues[varName] = Tuple.Create(dataset.GetDoubleValue(varName, t), Vector.Zero);
           }
@@ -656,5 +680,5 @@
         ) {
 
-      switch(node.Symbol.Name) {
+      switch (node.Symbol.Name) {
         case "+": {
             var l = InterpretRec(node.GetSubtree(0), variableValues, nodeIdx, parameterValues); // TODO capture all parameters into a state type for interpretation
@@ -681,5 +705,5 @@
 
             // protected division
-            if(r.Item1.IsAlmost(0.0)) {
+            if (r.Item1.IsAlmost(0.0)) {
               return Tuple.Create(0.0, Vector.Zero);
             } else {
@@ -690,7 +714,21 @@
             }
           }
+        case "sin": {
+            var x = InterpretRec(node.GetSubtree(0), variableValues, nodeIdx, parameterValues);
+            return Tuple.Create(
+              Math.Sin(x.Item1),
+              Vector.Cos(x.Item2) * x.Item2
+            );
+          }
+        case "cos": {
+            var x = InterpretRec(node.GetSubtree(0), variableValues, nodeIdx, parameterValues);
+            return Tuple.Create(
+              Math.Cos(x.Item1),
+              -Vector.Sin(x.Item2) * x.Item2
+            );
+          }
         default: {
             // distinguish other cases
-            if(IsConstantNode(node)) {
+            if (IsConstantNode(node)) {
               var vArr = new double[parameterValues.Length]; // backing array for vector
               vArr[nodeIdx[node]] = 1.0;
@@ -724,11 +762,11 @@
     private void RegisterEventHandlers() {
       ProblemDataParameter.ValueChanged += ProblemDataParameter_ValueChanged;
-      if(ProblemDataParameter.Value != null) ProblemDataParameter.Value.Changed += ProblemData_Changed;
+      if (ProblemDataParameter.Value != null) ProblemDataParameter.Value.Changed += ProblemData_Changed;
 
       TargetVariablesParameter.ValueChanged += TargetVariablesParameter_ValueChanged;
-      if(TargetVariablesParameter.Value != null) TargetVariablesParameter.Value.CheckedItemsChanged += CheckedTargetVariablesChanged;
+      if (TargetVariablesParameter.Value != null) TargetVariablesParameter.Value.CheckedItemsChanged += CheckedTargetVariablesChanged;
 
       FunctionSetParameter.ValueChanged += FunctionSetParameter_ValueChanged;
-      if(FunctionSetParameter.Value != null) FunctionSetParameter.Value.CheckedItemsChanged += CheckedFunctionsChanged;
+      if (FunctionSetParameter.Value != null) FunctionSetParameter.Value.CheckedItemsChanged += CheckedFunctionsChanged;
 
       MaximumLengthParameter.Value.ValueChanged += MaximumLengthChanged;
@@ -775,5 +813,5 @@
       UpdateTargetVariables();        // implicitly updates other dependent parameters
       var handler = ProblemDataChanged;
-      if(handler != null) handler(this, EventArgs.Empty);
+      if (handler != null) handler(this, EventArgs.Empty);
     }
 
@@ -792,4 +830,6 @@
       l.Add(new StringValue("%").AsReadOnly());
       l.Add(new StringValue("-").AsReadOnly());
+      l.Add(new StringValue("sin").AsReadOnly());
+      l.Add(new StringValue("cos").AsReadOnly());
       return l.AsReadOnly();
     }
@@ -808,5 +848,5 @@
       var newVariablesList = new CheckedItemCollection<StringValue>(ProblemData.Dataset.VariableNames.Select(str => new StringValue(str).AsReadOnly()).ToArray()).AsReadOnly();
       var matchingItems = newVariablesList.Where(item => currentlySelectedVariables.Contains(item.Value)).ToArray();
-      foreach(var matchingItem in matchingItems) {
+      foreach (var matchingItem in matchingItems) {
         newVariablesList.SetItemCheckedState(matchingItem, true);
       }
@@ -817,8 +857,8 @@
       var encoding = new MultiEncoding();
       var g = CreateGrammar();
-      foreach(var targetVar in TargetVariables.CheckedItems) {
+      foreach (var targetVar in TargetVariables.CheckedItems) {
         encoding = encoding.Add(new SymbolicExpressionTreeEncoding(targetVar + "_tree", g, MaximumLength, MaximumLength)); // only limit by length
       }
-      for(int i = 1; i <= NumberOfLatentVariables; i++) {
+      for (int i = 1; i <= NumberOfLatentVariables; i++) {
         encoding = encoding.Add(new SymbolicExpressionTreeEncoding("λ" + i + "_tree", g, MaximumLength, MaximumLength));
       }
@@ -838,5 +878,5 @@
       //}, 1, 1);
 
-      foreach(var variableName in ProblemData.AllowedInputVariables.Union(TargetVariables.CheckedItems.Select(i => i.Value)))
+      foreach (var variableName in ProblemData.AllowedInputVariables.Union(TargetVariables.CheckedItems.Select(i => i.Value)))
         g.AddTerminalSymbol(variableName);
 
@@ -844,10 +884,10 @@
       // we generate multiple symbols to balance the probability for selecting a numeric parameter in the generation of random trees
       var numericConstantsFactor = 2.0;
-      for(int i = 0; i < numericConstantsFactor * (ProblemData.AllowedInputVariables.Count() + TargetVariables.CheckedItems.Count()); i++) {
+      for (int i = 0; i < numericConstantsFactor * (ProblemData.AllowedInputVariables.Count() + TargetVariables.CheckedItems.Count()); i++) {
         g.AddTerminalSymbol("θ" + i); // numeric parameter for which the value is optimized using AutoDiff
       }
 
       // generate symbols for latent variables
-      for(int i = 1; i <= NumberOfLatentVariables; i++) {
+      for (int i = 1; i <= NumberOfLatentVariables; i++) {
         g.AddTerminalSymbol("λ" + i); // numeric parameter for which the value is optimized using AutoDiff
       }
