Index: /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/HeuristicLab.Problems.DynamicalSystemsModelling-3.3.csproj
===================================================================
--- /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/HeuristicLab.Problems.DynamicalSystemsModelling-3.3.csproj	(revision 16267)
+++ /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/HeuristicLab.Problems.DynamicalSystemsModelling-3.3.csproj	(revision 16268)
@@ -112,4 +112,5 @@
   <ItemGroup>
     <Compile Include="CVODES.cs" />
+    <Compile Include="OdeParameterIdentification.cs" />
     <Compile Include="Plugin.cs" />
     <Compile Include="Problem.cs" />
@@ -124,4 +125,8 @@
   </ItemGroup>
   <ItemGroup>
+    <ProjectReference Include="..\..\HeuristicLab.Algorithms.DataAnalysis\3.4\HeuristicLab.Algorithms.DataAnalysis-3.4.csproj">
+      <Project>{2E782078-FA81-4B70-B56F-74CE38DAC6C8}</Project>
+      <Name>HeuristicLab.Algorithms.DataAnalysis-3.4</Name>
+    </ProjectReference>
     <ProjectReference Include="..\..\HeuristicLab.Analysis\3.3\HeuristicLab.Analysis-3.3.csproj">
       <Project>{887425B4-4348-49ED-A457-B7D2C26DDBF9}</Project>
Index: /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/OdeParameterIdentification.cs
===================================================================
--- /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/OdeParameterIdentification.cs	(revision 16268)
+++ /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/OdeParameterIdentification.cs	(revision 16268)
@@ -0,0 +1,249 @@
+﻿#region License Information
+/* HeuristicLab
+ * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ *
+ * This file is part of HeuristicLab.
+ *
+ * HeuristicLab is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * HeuristicLab is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
+ */
+#endregion
+
+using System;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Algorithms.DataAnalysis;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+using HeuristicLab.Random;
+
+namespace HeuristicLab.Problems.DynamicalSystemsModelling {
+  [Item("OdeParameterIdentification", "TODO")]
+  [Creatable(CreatableAttribute.Categories.DataAnalysisRegression, Priority = 120)]
+  [StorableClass]
+  public sealed class OdeParameterIdentification : FixedDataAnalysisAlgorithm<Problem> {
+    private const string RegressionSolutionResultName = "Regression solution";
+    private const string ModelStructureParameterName = "Model structure";
+    private const string IterationsParameterName = "Iterations";
+    private const string RestartsParameterName = "Restarts";
+    private const string SetSeedRandomlyParameterName = "SetSeedRandomly";
+    private const string SeedParameterName = "Seed";
+    private const string InitParamsRandomlyParameterName = "InitializeParametersRandomly";
+
+    public IValueParameter<StringArray> ModelStructureParameter {
+      get { return (IValueParameter<StringArray>)Parameters[ModelStructureParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> IterationsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[IterationsParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> SetSeedRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[SetSeedRandomlyParameterName]; }
+    }
+
+    public IFixedValueParameter<IntValue> SeedParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[SeedParameterName]; }
+    }
+
+    public IFixedValueParameter<IntValue> RestartsParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[RestartsParameterName]; }
+    }
+
+    public IFixedValueParameter<BoolValue> InitParametersRandomlyParameter {
+      get { return (IFixedValueParameter<BoolValue>)Parameters[InitParamsRandomlyParameterName]; }
+    }
+
+    public StringArray ModelStructure {
+      get { return ModelStructureParameter.Value; }
+      set { ModelStructureParameter.Value = value; }
+    }
+
+    public int Iterations {
+      get { return IterationsParameter.Value.Value; }
+      set { IterationsParameter.Value.Value = value; }
+    }
+
+    public int Restarts {
+      get { return RestartsParameter.Value.Value; }
+      set { RestartsParameter.Value.Value = value; }
+    }
+
+    public int Seed {
+      get { return SeedParameter.Value.Value; }
+      set { SeedParameter.Value.Value = value; }
+    }
+
+    public bool SetSeedRandomly {
+      get { return SetSeedRandomlyParameter.Value.Value; }
+      set { SetSeedRandomlyParameter.Value.Value = value; }
+    }
+
+    public bool InitializeParametersRandomly {
+      get { return InitParametersRandomlyParameter.Value.Value; }
+      set { InitParametersRandomlyParameter.Value.Value = value; }
+    }
+
+    [StorableConstructor]
+    private OdeParameterIdentification(bool deserializing) : base(deserializing) { }
+    private OdeParameterIdentification(OdeParameterIdentification original, Cloner cloner)
+      : base(original, cloner) {
+    }
+    public OdeParameterIdentification()
+      : base() {
+      Problem = new Problem();
+      Parameters.Add(new ValueParameter<StringArray>(ModelStructureParameterName, "The function for which the parameters must be fit (only numeric constants are tuned).", new StringArray(new string[] { "1.0 * x*x + 0.0" })));
+      Parameters.Add(new FixedValueParameter<IntValue>(IterationsParameterName, "The maximum number of iterations for constants optimization.", new IntValue(200)));
+      Parameters.Add(new FixedValueParameter<IntValue>(RestartsParameterName, "The number of independent random restarts (>0)", new IntValue(10)));
+      Parameters.Add(new FixedValueParameter<IntValue>(SeedParameterName, "The PRNG seed value.", new IntValue()));
+      Parameters.Add(new FixedValueParameter<BoolValue>(SetSeedRandomlyParameterName, "Switch to determine if the random number seed should be initialized randomly.", new BoolValue(true)));
+      Parameters.Add(new FixedValueParameter<BoolValue>(InitParamsRandomlyParameterName, "Switch to determine if the real-valued model parameters should be initialized randomly in each restart.", new BoolValue(false)));
+
+      SetParameterHiddenState();
+
+      InitParametersRandomlyParameter.Value.ValueChanged += (sender, args) => {
+        SetParameterHiddenState();
+      };
+    }
+
+    private void SetParameterHiddenState() {
+      var hide = !InitializeParametersRandomly;
+      RestartsParameter.Hidden = hide;
+      SeedParameter.Hidden = hide;
+      SetSeedRandomlyParameter.Hidden = hide;
+    }
+
+    [StorableHook(HookType.AfterDeserialization)]
+    private void AfterDeserialization() {
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new OdeParameterIdentification(this, cloner);
+    }
+
+    #region nonlinear regression
+    protected override void Run(CancellationToken cancellationToken) {
+      IRegressionSolution bestSolution = null;
+      if (SetSeedRandomly) Seed = (new System.Random()).Next();
+      var rand = new MersenneTwister((uint)Seed);
+      if (InitializeParametersRandomly) {
+        throw new NotImplementedException();
+        // var qualityTable = new DataTable("RMSE table");
+        // qualityTable.VisualProperties.YAxisLogScale = true;
+        // var trainRMSERow = new DataRow("RMSE (train)");
+        // trainRMSERow.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+        // var testRMSERow = new DataRow("RMSE test");
+        // testRMSERow.VisualProperties.ChartType = DataRowVisualProperties.DataRowChartType.Points;
+        // 
+        // qualityTable.Rows.Add(trainRMSERow);
+        // qualityTable.Rows.Add(testRMSERow);
+        // Results.Add(new Result(qualityTable.Name, qualityTable.Name + " for all restarts", qualityTable));
+
+        // CreateSolution(Problem, ModelStructure.ToArray(), Iterations, rand);
+        // 
+        // for (int r = 0; r < Restarts; r++) {
+        //   CreateSolution(Problem, ModelStructure.ToArray(), Iterations, rand);
+        //   trainRMSERow.Values.Add(solution.TrainingRootMeanSquaredError);
+        //   testRMSERow.Values.Add(solution.TestRootMeanSquaredError);
+        //   if (solution.TrainingRootMeanSquaredError < bestSolution.TrainingRootMeanSquaredError) {
+        //     bestSolution = solution;
+        //   }
+        // }
+      } else {
+        CreateSolution(Problem, ModelStructure.ToArray(), Iterations, rand);
+      }
+
+      // Results.Add(new Result(RegressionSolutionResultName, "The nonlinear regression solution.", bestSolution));
+      // Results.Add(new Result("Root mean square error (train)", "The root of the mean of squared errors of the regression solution on the training set.", new DoubleValue(bestSolution.TrainingRootMeanSquaredError)));
+      // Results.Add(new Result("Root mean square error (test)", "The root of the mean of squared errors of the regression solution on the test set.", new DoubleValue(bestSolution.TestRootMeanSquaredError)));
+
+    }
+
+    public void CreateSolution(Problem problem, string[] modelStructure, int maxIterations, IRandom rand) {
+      var parser = new InfixExpressionParser();
+      var trees = modelStructure.Select(expr => Convert(parser.Parse(expr))).ToArray();
+      var names = problem.Encoding.Encodings.Select(enc => enc.Name).ToArray();
+      if (trees.Length != names.Length) throw new ArgumentException("The number of expressions must match the number of target variables exactly");
+
+      var scope = new Scope();
+      for (int i = 0; i < names.Length; i++) {
+        scope.Variables.Add(new Core.Variable(names[i], trees[i]));
+      }
+      var ind = problem.Encoding.GetIndividual(scope);
+      var quality = problem.Evaluate(ind, rand);
+      problem.Analyze(new[] { ind }, new[] { quality }, Results, rand);
+    }
+
+    private ISymbolicExpressionTree Convert(ISymbolicExpressionTree tree) {
+      return new SymbolicExpressionTree(Convert(tree.Root));
+    }
+
+    private ISymbolicExpressionTreeNode Convert(ISymbolicExpressionTreeNode node) {
+      if (node.Symbol is ProgramRootSymbol) {
+        var child = Convert(node.GetSubtree(0));
+        node.RemoveSubtree(0);
+        node.AddSubtree(child);
+        return node;
+      } else if (node.Symbol is StartSymbol) {
+        var child = Convert(node.GetSubtree(0));
+        node.RemoveSubtree(0);
+        node.AddSubtree(child);
+        return node;
+      } else if (node.Symbol is Addition) {
+        var children = node.Subtrees.Select(st => Convert(st)).ToArray();
+        return Make("+", children);
+      } else if (node.Symbol is Subtraction) {
+        var children = node.Subtrees.Select(st => Convert(st)).ToArray();
+        return Make("-", children);
+      } else if (node.Symbol is Multiplication) {
+        var children = node.Subtrees.Select(st => Convert(st)).ToArray();
+        return Make("*", children);
+      } else if (node.Symbol is Division) {
+        var children = node.Subtrees.Select(st => Convert(st)).ToArray();
+        if (children.Length == 1) {
+          return Make("%", new[] { new SimpleSymbol("θ", 0).CreateTreeNode(), children[0] });
+        } else if (children.Length != 2) throw new ArgumentException("Division is not supported for multiple arguments");
+        else return Make("%", children);
+      } else if (node.Symbol is Constant) {
+        return new SimpleSymbol("θ", 0).CreateTreeNode();
+      } else if (node.Symbol is DataAnalysis.Symbolic.Variable) {
+        var varNode = node as VariableTreeNode;
+        if (!varNode.Weight.IsAlmost(1.0)) throw new ArgumentException("Variable weights are not supported");
+        return new SimpleSymbol(varNode.VariableName, 0).CreateTreeNode();
+      } else throw new ArgumentException("Unsupported symbol: " + node.Symbol.Name);
+    }
+
+    private ISymbolicExpressionTreeNode Make(string op, ISymbolicExpressionTreeNode[] children) {
+      var s = new SimpleSymbol(op, 2).CreateTreeNode();
+      var c0 = children[0];
+      var c1 = children[1];
+      s.AddSubtree(c0);
+      s.AddSubtree(c1);
+      for (int i = 2; i < children.Length; i++) {
+        var sn = new SimpleSymbol(op, 2).CreateTreeNode();
+        sn.AddSubtree(s);
+        sn.AddSubtree(children[i]);
+        s = sn;
+      }
+      return s;
+    }
+    #endregion
+  }
+}
Index: /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs
===================================================================
--- /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs	(revision 16267)
+++ /branches/2925_AutoDiffForDynamicalModels/HeuristicLab.Problems.DynamicalSystemsModelling/3.3/Problem.cs	(revision 16268)
@@ -69,9 +69,9 @@
       get { return (IValueParameter<IRegressionProblemData>)Parameters[ProblemDataParameterName]; }
     }
-    public IValueParameter<ReadOnlyCheckedItemCollection<StringValue>> TargetVariablesParameter {
-      get { return (IValueParameter<ReadOnlyCheckedItemCollection<StringValue>>)Parameters[TargetVariablesParameterName]; }
-    }
-    public IValueParameter<ReadOnlyCheckedItemCollection<StringValue>> FunctionSetParameter {
-      get { return (IValueParameter<ReadOnlyCheckedItemCollection<StringValue>>)Parameters[FunctionSetParameterName]; }
+    public IValueParameter<ReadOnlyCheckedItemList<StringValue>> TargetVariablesParameter {
+      get { return (IValueParameter<ReadOnlyCheckedItemList<StringValue>>)Parameters[TargetVariablesParameterName]; }
+    }
+    public IValueParameter<ReadOnlyCheckedItemList<StringValue>> FunctionSetParameter {
+      get { return (IValueParameter<ReadOnlyCheckedItemList<StringValue>>)Parameters[FunctionSetParameterName]; }
     }
     public IFixedValueParameter<IntValue> MaximumLengthParameter {
@@ -105,9 +105,9 @@
     IDataAnalysisProblemData IDataAnalysisProblem.ProblemData { get { return ProblemData; } }
 
-    public ReadOnlyCheckedItemCollection<StringValue> TargetVariables {
+    public ReadOnlyCheckedItemList<StringValue> TargetVariables {
       get { return TargetVariablesParameter.Value; }
     }
 
-    public ReadOnlyCheckedItemCollection<StringValue> FunctionSet {
+    public ReadOnlyCheckedItemList<StringValue> FunctionSet {
       get { return FunctionSetParameter.Value; }
     }
@@ -171,9 +171,9 @@
     public Problem()
       : base() {
-      var targetVariables = new CheckedItemCollection<StringValue>().AsReadOnly(); // HACK: it would be better to provide a new class derived from IDataAnalysisProblem
+      var targetVariables = new CheckedItemList<StringValue>().AsReadOnly(); // HACK: it would be better to provide a new class derived from IDataAnalysisProblem
       var functions = CreateFunctionSet();
       Parameters.Add(new ValueParameter<IRegressionProblemData>(ProblemDataParameterName, "The data captured from the dynamical system. Use CSV import functionality to import data.", new RegressionProblemData()));
-      Parameters.Add(new ValueParameter<ReadOnlyCheckedItemCollection<StringValue>>(TargetVariablesParameterName, "Target variables (overrides setting in ProblemData)", targetVariables));
-      Parameters.Add(new ValueParameter<ReadOnlyCheckedItemCollection<StringValue>>(FunctionSetParameterName, "The list of allowed functions", functions));
+      Parameters.Add(new ValueParameter<ReadOnlyCheckedItemList<StringValue>>(TargetVariablesParameterName, "Target variables (overrides setting in ProblemData)", targetVariables));
+      Parameters.Add(new ValueParameter<ReadOnlyCheckedItemList<StringValue>>(FunctionSetParameterName, "The list of allowed functions", functions));
       Parameters.Add(new FixedValueParameter<IntValue>(MaximumLengthParameterName, "The maximally allowed length of each expression. Set to a small value (5 - 25). Default = 10", new IntValue(10)));
       Parameters.Add(new FixedValueParameter<IntValue>(MaximumParameterOptimizationIterationsParameterName, "The maximum number of iterations for optimization of parameters (using L-BFGS). More iterations makes the algorithm slower, fewer iterations might prevent convergence in the optimization scheme. Default = 100", new IntValue(100)));
@@ -235,5 +235,5 @@
       var rows = episodes.SelectMany(e => Enumerable.Range(e.Start, e.End - e.Start)).ToArray();
       var problemData = ProblemData;
-      var targetVars = TargetVariables.CheckedItems.Select(i => i.Value).ToArray();
+      var targetVars = TargetVariables.CheckedItems.OrderBy(i => i.Index).Select(i => i.Value.Value).ToArray();
       var latentVariables = Enumerable.Range(1, NumberOfLatentVariables).Select(i => "λ" + i).ToArray(); // TODO: must coincide with the variables which are actually defined in the grammar and also for which we actually have trees
       var targetValues = new double[rows.Length, targetVars.Length];
@@ -258,5 +258,6 @@
         }
       }
-      var theta = paramNodes.Select(_ => random.NextDouble() * 2.0 - 1.0).ToArray(); // init params randomly from Unif(-1,1)
+      // init params randomly from Unif(-1e-5, 1e-5)
+      var theta = paramNodes.Select(_ => random.NextDouble() * 2.0e-5 - 1.0e-5).ToArray();
 
       optTheta = new double[0];
@@ -266,4 +267,5 @@
         alglib.minlbfgscreate(Math.Min(theta.Length, 5), theta, out state);
         alglib.minlbfgssetcond(state, 0.0, 0.0, 0.0, MaximumParameterOptimizationIterations);
+        alglib.minlbfgssetgradientcheck(state, 1e-6);
         alglib.minlbfgsoptimize(state, EvaluateObjectiveAndGradient, null,
           new object[] { trees, targetVars, problemData, targetValues, episodes.ToArray(), NumericIntegrationSteps, latentVariables, OdeSolver }); //TODO: create a type
@@ -386,5 +388,5 @@
 
       var problemData = ProblemData;
-      var targetVars = TargetVariables.CheckedItems.Select(i => i.Value).ToArray();
+      var targetVars = TargetVariables.CheckedItems.OrderBy(i => i.Index).Select(i => i.Value.Value).ToArray();
       var latentVariables = Enumerable.Range(1, NumberOfLatentVariables).Select(i => "λ" + i).ToArray(); // TODO: must coincide with the variables which are actually defined in the grammar and also for which we actually have trees
 
@@ -741,4 +743,5 @@
                       var varName = n.Symbol.Name;
                       var varIdx = Array.IndexOf(calculatedVariables, varName); // TODO: perf!
+                      if (varIdx < 0) throw new InvalidProgramException();
                       var y_i = CVODES.NV_Get_Ith_S(y, (long)varIdx);
                       nodeValues.Add(n, Tuple.Create(y_i, Vector.Zero)); // no gradient needed
@@ -782,4 +785,6 @@
               var varName = n.Symbol.Name;
               var varIdx = Array.IndexOf(calculatedVariables, varName); // TODO: perf!
+              if (varIdx < 0) throw new InvalidProgramException();
+
               var y_i = CVODES.NV_Get_Ith_S(y, (long)varIdx);
               var gArr = new double[CVODES.NV_LENGTH_S(y)]; // backing array
@@ -829,4 +834,6 @@
                 var varName = n.Symbol.Name;
                 var varIdx = Array.IndexOf(calculatedVariables, varName); // TODO: perf!
+                if (varIdx < 0) throw new InvalidProgramException();
+
                 var y_i = CVODES.NV_Get_Ith_S(y, (long)varIdx);
                 var gArr = new double[d]; // backing array
@@ -1053,5 +1060,5 @@
     }
 
-    private void CheckedFunctionsChanged(object sender, CollectionItemsChangedEventArgs<StringValue> e) {
+    private void CheckedFunctionsChanged(object sender, CollectionItemsChangedEventArgs<IndexedItem<StringValue>> e) {
       UpdateGrammarAndEncoding();
     }
@@ -1061,5 +1068,5 @@
     }
 
-    private void CheckedTargetVariablesChanged(object sender, CollectionItemsChangedEventArgs<StringValue> e) {
+    private void CheckedTargetVariablesChanged(object sender, CollectionItemsChangedEventArgs<IndexedItem<StringValue>> e) {
       UpdateGrammarAndEncoding();
     }
@@ -1090,6 +1097,6 @@
     }
 
-    private ReadOnlyCheckedItemCollection<StringValue> CreateFunctionSet() {
-      var l = new CheckedItemCollection<StringValue>();
+    private ReadOnlyCheckedItemList<StringValue> CreateFunctionSet() {
+      var l = new CheckedItemList<StringValue>();
       l.Add(new StringValue("+").AsReadOnly());
       l.Add(new StringValue("*").AsReadOnly());
@@ -1113,7 +1120,10 @@
 
     private void UpdateTargetVariables() {
-      var currentlySelectedVariables = TargetVariables.CheckedItems.Select(i => i.Value).ToArray();
-
-      var newVariablesList = new CheckedItemCollection<StringValue>(ProblemData.Dataset.VariableNames.Select(str => new StringValue(str).AsReadOnly()).ToArray()).AsReadOnly();
+      var currentlySelectedVariables = TargetVariables.CheckedItems
+        .OrderBy(i => i.Index)
+        .Select(i => i.Value.Value)
+        .ToArray();
+
+      var newVariablesList = new CheckedItemList<StringValue>(ProblemData.Dataset.VariableNames.Select(str => new StringValue(str).AsReadOnly()).ToArray()).AsReadOnly();
       var matchingItems = newVariablesList.Where(item => currentlySelectedVariables.Contains(item.Value)).ToArray();
       foreach (var matchingItem in matchingItems) {
@@ -1138,5 +1148,5 @@
       // whenever ProblemData is changed we create a new grammar with the necessary symbols
       var g = new SimpleSymbolicExpressionGrammar();
-      g.AddSymbols(FunctionSet.CheckedItems.Select(i => i.Value).ToArray(), 2, 2);
+      g.AddSymbols(FunctionSet.CheckedItems.OrderBy(i => i.Index).Select(i => i.Value.Value).ToArray(), 2, 2);
 
       // TODO
@@ -1147,5 +1157,5 @@
       //}, 1, 1);
 
-      foreach (var variableName in ProblemData.AllowedInputVariables.Union(TargetVariables.CheckedItems.Select(i => i.Value)))
+      foreach (var variableName in ProblemData.AllowedInputVariables.Union(TargetVariables.CheckedItems.Select(i => i.Value.Value)))
         g.AddTerminalSymbol(variableName);
 
