Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/GrammaticalOptimization.sln
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/GrammaticalOptimization.sln	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/GrammaticalOptimization.sln	(revision 12290)
@@ -0,0 +1,124 @@
+﻿
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 2013
+VisualStudioVersion = 12.0.31101.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Problems.GrammaticalOptimization", "HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.csproj", "{CB9DCCF6-667E-4A13-B82D-DBD6B45A045E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.GrammaticalOptimization", "HeuristicLab.Algorithms.GrammaticalOptimization\HeuristicLab.Algorithms.GrammaticalOptimization.csproj", "{EEA07488-1A51-412A-A52C-53B754A628B3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Main", "Main\Main.csproj", "{524CBD70-6F99-46AF-AD00-A0BC9FD1175B}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.Bandits", "HeuristicLab.Algorithms.Bandits\HeuristicLab.Algorithms.Bandits.csproj", "{24408F7D-EE0F-4886-A08B-EC324D662E47}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Common", "HeuristicLab.Common\HeuristicLab.Common.csproj", "{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Problems.GrammaticalOptimization.SymbReg", "HeuristicLab.Problems.GrammaticalOptimization.SymbReg\HeuristicLab.Problems.GrammaticalOptimization.SymbReg.csproj", "{17A7A380-86CE-482D-8D22-CBD70CC97F0D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.GeneticProgramming", "HeuristicLab.Algorithms.GeneticProgramming\HeuristicLab.Algorithms.GeneticProgramming.csproj", "{14BEC23F-63FD-4954-B8AE-E2F4962E9B57}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Distributions", "HeuristicLab.Distributions\HeuristicLab.Distributions.csproj", "{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Problems.Bandits", "HeuristicLab.Problems.Bandits\HeuristicLab.Problems.Bandits.csproj", "{E8138227-0C64-4E85-B676-21D3B97F254F}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Test", "Test\Test.csproj", "{8B4C199E-C08B-479E-9B82-6AE5B113459E}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HeuristicLab", "HeuristicLab", "{FE0B0799-C3D4-407C-9DD2-899BCDA7AB9A}"
+	ProjectSection(SolutionItems) = preProject
+		HeuristicLab\ALGLIB-3.7.0.dll = HeuristicLab\ALGLIB-3.7.0.dll
+		HeuristicLab\AutoDiff-1.0.dll = HeuristicLab\AutoDiff-1.0.dll
+		HeuristicLab\HeuristicLab.Algorithms.Bandits.dll = HeuristicLab\HeuristicLab.Algorithms.Bandits.dll
+		HeuristicLab\HeuristicLab.Algorithms.GeneticAlgorithm-3.3.dll = HeuristicLab\HeuristicLab.Algorithms.GeneticAlgorithm-3.3.dll
+		HeuristicLab\HeuristicLab.Algorithms.GeneticProgramming.dll = HeuristicLab\HeuristicLab.Algorithms.GeneticProgramming.dll
+		HeuristicLab\HeuristicLab.Algorithms.GrammaticalOptimization.dll = HeuristicLab\HeuristicLab.Algorithms.GrammaticalOptimization.dll
+		HeuristicLab\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3.dll = HeuristicLab\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3.dll
+		HeuristicLab\HeuristicLab.Analysis-3.3.dll = HeuristicLab\HeuristicLab.Analysis-3.3.dll
+		HeuristicLab\HeuristicLab.Collections-3.3.dll = HeuristicLab\HeuristicLab.Collections-3.3.dll
+		HeuristicLab\HeuristicLab.Common-3.3.dll = HeuristicLab\HeuristicLab.Common-3.3.dll
+		HeuristicLab\HeuristicLab.Common.dll = HeuristicLab\HeuristicLab.Common.dll
+		HeuristicLab\HeuristicLab.Common.Resources-3.3.dll = HeuristicLab\HeuristicLab.Common.Resources-3.3.dll
+		HeuristicLab\HeuristicLab.Core-3.3.dll = HeuristicLab\HeuristicLab.Core-3.3.dll
+		HeuristicLab\HeuristicLab.Data-3.3.dll = HeuristicLab\HeuristicLab.Data-3.3.dll
+		HeuristicLab\HeuristicLab.Distributions.dll = HeuristicLab\HeuristicLab.Distributions.dll
+		HeuristicLab\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.dll = HeuristicLab\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.dll
+		HeuristicLab\HeuristicLab.Operators-3.3.dll = HeuristicLab\HeuristicLab.Operators-3.3.dll
+		HeuristicLab\HeuristicLab.Operators.Programmable-3.3.dll = HeuristicLab\HeuristicLab.Operators.Programmable-3.3.dll
+		HeuristicLab\HeuristicLab.Optimization-3.3.dll = HeuristicLab\HeuristicLab.Optimization-3.3.dll
+		HeuristicLab\HeuristicLab.Optimization.Operators-3.3.dll = HeuristicLab\HeuristicLab.Optimization.Operators-3.3.dll
+		HeuristicLab\HeuristicLab.ParallelEngine-3.3.dll = HeuristicLab\HeuristicLab.ParallelEngine-3.3.dll
+		HeuristicLab\HeuristicLab.Parameters-3.3.dll = HeuristicLab\HeuristicLab.Parameters-3.3.dll
+		HeuristicLab\HeuristicLab.Persistence-3.3.dll = HeuristicLab\HeuristicLab.Persistence-3.3.dll
+		HeuristicLab\HeuristicLab.PluginInfrastructure-3.3.dll = HeuristicLab\HeuristicLab.PluginInfrastructure-3.3.dll
+		HeuristicLab\HeuristicLab.Problems.DataAnalysis-3.4.dll = HeuristicLab\HeuristicLab.Problems.DataAnalysis-3.4.dll
+		HeuristicLab\HeuristicLab.Problems.GrammaticalOptimization.dll = HeuristicLab\HeuristicLab.Problems.GrammaticalOptimization.dll
+		HeuristicLab\HeuristicLab.Problems.GrammaticalOptimization.SymbReg.dll = HeuristicLab\HeuristicLab.Problems.GrammaticalOptimization.SymbReg.dll
+		HeuristicLab\HeuristicLab.Problems.Instances-3.3.dll = HeuristicLab\HeuristicLab.Problems.Instances-3.3.dll
+		HeuristicLab\HeuristicLab.Problems.Instances.DataAnalysis-3.3.dll = HeuristicLab\HeuristicLab.Problems.Instances.DataAnalysis-3.3.dll
+		HeuristicLab\HeuristicLab.Random-3.3.dll = HeuristicLab\HeuristicLab.Random-3.3.dll
+		HeuristicLab\HeuristicLab.Selection-3.3.dll = HeuristicLab\HeuristicLab.Selection-3.3.dll
+		HeuristicLab\HeuristicLab.SequentialEngine-3.3.dll = HeuristicLab\HeuristicLab.SequentialEngine-3.3.dll
+		HeuristicLab\HeuristicLab.Tracing-3.3.dll = HeuristicLab\HeuristicLab.Tracing-3.3.dll
+	EndProjectSection
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.MonteCarloTreeSearch", "HeuristicLab.Algorithms.MonteCarloTreeSearch\HeuristicLab.Algorithms.MonteCarloTreeSearch.csproj", "{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D521D944-A271-4A74-8511-6608622509A4}"
+	ProjectSection(SolutionItems) = preProject
+		local.testsettings = local.testsettings
+	EndProjectSection
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{CB9DCCF6-667E-4A13-B82D-DBD6B45A045E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{CB9DCCF6-667E-4A13-B82D-DBD6B45A045E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{CB9DCCF6-667E-4A13-B82D-DBD6B45A045E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{CB9DCCF6-667E-4A13-B82D-DBD6B45A045E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{EEA07488-1A51-412A-A52C-53B754A628B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{EEA07488-1A51-412A-A52C-53B754A628B3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{EEA07488-1A51-412A-A52C-53B754A628B3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{EEA07488-1A51-412A-A52C-53B754A628B3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{524CBD70-6F99-46AF-AD00-A0BC9FD1175B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{524CBD70-6F99-46AF-AD00-A0BC9FD1175B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{524CBD70-6F99-46AF-AD00-A0BC9FD1175B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{524CBD70-6F99-46AF-AD00-A0BC9FD1175B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{24408F7D-EE0F-4886-A08B-EC324D662E47}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{24408F7D-EE0F-4886-A08B-EC324D662E47}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{24408F7D-EE0F-4886-A08B-EC324D662E47}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{24408F7D-EE0F-4886-A08B-EC324D662E47}.Release|Any CPU.Build.0 = Release|Any CPU
+		{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}.Release|Any CPU.Build.0 = Release|Any CPU
+		{17A7A380-86CE-482D-8D22-CBD70CC97F0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{17A7A380-86CE-482D-8D22-CBD70CC97F0D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{17A7A380-86CE-482D-8D22-CBD70CC97F0D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{17A7A380-86CE-482D-8D22-CBD70CC97F0D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{14BEC23F-63FD-4954-B8AE-E2F4962E9B57}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{14BEC23F-63FD-4954-B8AE-E2F4962E9B57}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{14BEC23F-63FD-4954-B8AE-E2F4962E9B57}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{14BEC23F-63FD-4954-B8AE-E2F4962E9B57}.Release|Any CPU.Build.0 = Release|Any CPU
+		{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E8138227-0C64-4E85-B676-21D3B97F254F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E8138227-0C64-4E85-B676-21D3B97F254F}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E8138227-0C64-4E85-B676-21D3B97F254F}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E8138227-0C64-4E85-B676-21D3B97F254F}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8B4C199E-C08B-479E-9B82-6AE5B113459E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8B4C199E-C08B-479E-9B82-6AE5B113459E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8B4C199E-C08B-479E-9B82-6AE5B113459E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8B4C199E-C08B-479E-9B82-6AE5B113459E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/BernoulliPolicyActionInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/BernoulliPolicyActionInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/BernoulliPolicyActionInfo.cs	(revision 12290)
@@ -0,0 +1,34 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class BernoulliPolicyActionInfo : IBanditPolicyActionInfo {
+    public int NumSuccess { get; private set; }
+    public int NumFailure { get; private set; }
+    public int Tries { get { return NumSuccess + NumFailure; } }
+    public double Value {
+      get {
+        return NumSuccess / (double)(Tries);
+      }
+    }
+    public void UpdateReward(double reward) {
+      //Debug.Assert(reward.IsAlmost(0.0) || reward.IsAlmost(1.0));
+
+      //if (reward.IsAlmost(1.0)) NumSuccess++;
+      if (reward > 0) NumSuccess++;
+      else NumFailure++;
+    }
+    public void Reset() {
+      NumSuccess = 0;
+      NumFailure = 0;
+    }
+    public void PrintStats() {
+      Console.WriteLine("expected value {0,5:F2}", Value);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/DefaultPolicyActionInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/DefaultPolicyActionInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/DefaultPolicyActionInfo.cs	(revision 12290)
@@ -0,0 +1,45 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  // stores information that is relevant for most of the policies
+  public class DefaultPolicyActionInfo : IBanditPolicyActionInfo {
+    public double SumReward { get; private set; }
+    public int Tries { get; private set; }
+    public double MaxReward { get; private set; }
+    private double avgValue = 0.0;
+    public double Value {
+      get {
+        return Tries > 0 ? avgValue : double.PositiveInfinity;
+      }
+    }
+    public DefaultPolicyActionInfo() {
+      MaxReward = 0.0;
+    }
+
+    public void UpdateReward(double reward) {
+      Tries++;
+      SumReward += reward;
+      MaxReward = Math.Max(MaxReward, reward);
+      var delta = reward - avgValue;
+      //var alpha = 0.01;
+      var alpha = Math.Max(1.0/Tries, 0.01);
+      avgValue = avgValue + alpha * delta;
+    }
+
+    public void Reset() {
+      SumReward = 0.0;
+      Tries = 0;
+      MaxReward = 0.0;
+      avgValue = 0.0;
+    }
+
+    public override string ToString() {
+      return string.Format("{0:F3} {1:F3} {2}", Value, MaxReward, Tries);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/MeanAndVariancePolicyActionInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/MeanAndVariancePolicyActionInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/MeanAndVariancePolicyActionInfo.cs	(revision 12290)
@@ -0,0 +1,33 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class MeanAndVariancePolicyActionInfo : IBanditPolicyActionInfo {
+    private OnlineMeanAndVarianceEstimator estimator = new OnlineMeanAndVarianceEstimator();
+    public int Tries { get { return estimator.N; } }
+    public double SumReward { get { return estimator.Sum; } }
+    public double AvgReward { get { return estimator.Avg; } }
+    public double RewardVariance { get { return estimator.Variance; } }
+    public double Value {
+      get {
+        return AvgReward;
+      }
+    }
+
+    public void UpdateReward(double reward) {
+      estimator.UpdateReward(reward);
+    }
+
+    public void Reset() {
+      estimator.Reset();
+    }
+
+    public override string ToString() {
+      return string.Format("{0:N3} {1,3}", AvgReward, Tries);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ModelPolicyActionInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ModelPolicyActionInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/ActionInfos/ModelPolicyActionInfo.cs	(revision 12290)
@@ -0,0 +1,41 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  // uses a statistical model to sample and update posterior distribution p(Reward | Data)
+  public class ModelPolicyActionInfo : IBanditPolicyActionInfo {
+    private readonly IModel model;
+    public double Value {
+      get {
+        return model.Sample(new Random());
+      }
+    }
+
+    public int Tries { get; private set; }
+    public ModelPolicyActionInfo(IModel model) {
+      this.model = model;
+    }
+
+    public void UpdateReward(double reward) {
+      Tries++;
+      model.Update(reward);
+    }
+
+    public double SampleExpectedReward(Random random) {
+      return model.Sample(random);
+    }
+
+    public void Reset() {
+      Tries = 0;
+      model.Reset();
+    }
+
+    public override string ToString() {
+      return string.Format("model {1}", model);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/HeuristicLab.Algorithms.Bandits.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/HeuristicLab.Algorithms.Bandits.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/HeuristicLab.Algorithms.Bandits.csproj	(revision 12290)
@@ -0,0 +1,77 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{24408F7D-EE0F-4886-A08B-EC324D662E47}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Algorithms.Bandits</RootNamespace>
+    <AssemblyName>HeuristicLab.Algorithms.Bandits</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ActionInfos\BernoulliPolicyActionInfo.cs" />
+    <Compile Include="ActionInfos\DefaultPolicyActionInfo.cs" />
+    <Compile Include="ActionInfos\MeanAndVariancePolicyActionInfo.cs" />
+    <Compile Include="ActionInfos\ModelPolicyActionInfo.cs" />
+    <Compile Include="IBanditPolicy.cs" />
+    <Compile Include="IBanditPolicyActionInfo.cs" />
+    <Compile Include="Policies\ActiveLearningPolicy.cs" />
+    <Compile Include="Policies\BernoulliThompsonSamplingPolicy.cs" />
+    <Compile Include="Policies\BoltzmannExplorationPolicy.cs" />
+    <Compile Include="Policies\ChernoffIntervalEstimationPolicy.cs" />
+    <Compile Include="Policies\EpsGreedyPolicy.cs" />
+    <Compile Include="Policies\GaussianThompsonSamplingPolicy.cs" />
+    <Compile Include="Policies\GenericThompsonSamplingPolicy.cs" />
+    <Compile Include="Policies\ModifiedUCTPolicy.cs" />
+    <Compile Include="Policies\RandomPolicy.cs" />
+    <Compile Include="Policies\ThresholdAscentPolicy.cs" />
+    <Compile Include="Policies\UCB1Policy.cs" />
+    <Compile Include="Policies\UCB1TunedPolicy.cs" />
+    <Compile Include="Policies\UCBNormalPolicy.cs" />
+    <Compile Include="Policies\UCTPolicy.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Distributions\HeuristicLab.Distributions.csproj">
+      <Project>{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}</Project>
+      <Name>HeuristicLab.Distributions</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicy.cs	(revision 12290)
@@ -0,0 +1,13 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  // this interface represents a policy for bandit problems
+  public interface IBanditPolicy {
+    int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actions);
+    IBanditPolicyActionInfo CreateActionInfo();
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicyActionInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicyActionInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/IBanditPolicyActionInfo.cs	(revision 12290)
@@ -0,0 +1,11 @@
+﻿namespace HeuristicLab.Algorithms.Bandits {
+  public interface IBanditPolicyActionInfo {
+    //bool Disabled { get; }
+    double Value { get; }
+    int Tries { get; }
+    void UpdateReward(double reward);
+    //void Disable(double reward);
+    // reset causes the state of the action to be reinitialized (as after constructor-call)
+    void Reset();
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ActiveLearningPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ActiveLearningPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ActiveLearningPolicy.cs	(revision 12290)
@@ -0,0 +1,54 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class ActiveLearningPolicy : IBanditPolicy {
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+      int totalTries = myActionInfos.Sum(a => a.Tries);
+      const double delta = 0.1;
+      int k = myActionInfos.Count();
+      var bestActions = new List<int>();
+      var us = new List<double>();
+      var ls = new List<double>();
+      int aIdx = -1;
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        double u;
+        double l;
+        if (aInfo.Tries == 0) {
+          u = double.PositiveInfinity;
+          l = double.NegativeInfinity;
+        } else {
+          q = aInfo.SumReward / aInfo.Tries;
+          var b = Math.Sqrt(Math.Log(2.0 * k * totalTries / delta) / (2.0 * aInfo.Tries));
+          u = q + 0.5 * b;
+          l = q - 0.5 * b;
+        }
+        bestActions.Add(aIdx);
+        us.Add(u);
+        ls.Add(l);
+      }
+      var active = new List<int>();
+      var maxL = ls.Max();
+      for (int i = 0; i < us.Count; i++) {
+        if (us[i] >= maxL) active.Add(bestActions[i]);
+      }
+      Debug.Assert(active.Any());
+      return active.SelectRandom(random);
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+    public override string ToString() {
+      return "ActiveLearningPolicy";
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BernoulliThompsonSamplingPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BernoulliThompsonSamplingPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BernoulliThompsonSamplingPolicy.cs	(revision 12290)
@@ -0,0 +1,42 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class BernoulliThompsonSamplingPolicy : IBanditPolicy {
+    // parameters of beta prior distribution
+    private readonly double alpha = 1.0;
+    private readonly double beta = 1.0;
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<BernoulliPolicyActionInfo>();
+      int bestAction = -1;
+      double maxTheta = double.NegativeInfinity;
+      var aIdx = -1;
+
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        var theta = Rand.BetaRand(random, aInfo.NumSuccess + alpha, aInfo.NumFailure + beta);
+        if (theta > maxTheta) {
+          maxTheta = theta;
+          bestAction = aIdx;
+        }
+      }
+      Debug.Assert(bestAction > -1);
+      return bestAction;
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new BernoulliPolicyActionInfo();
+    }
+
+
+    public override string ToString() {
+      return "BernoulliThompsonSamplingPolicy";
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/BoltzmannExplorationPolicy.cs	(revision 12290)
@@ -0,0 +1,50 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  // also called softmax policy
+  public class BoltzmannExplorationPolicy : IBanditPolicy {
+    private readonly double beta;
+
+    public BoltzmannExplorationPolicy(double beta)  { 
+      if (beta < 0) throw new ArgumentException();
+      this.beta = beta;
+    }
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      Debug.Assert(actionInfos.Any());
+
+      // select best
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+
+      // try any of the untries actions randomly
+      // for RoyalSequence it is much better to select the actions in the order of occurrence (all terminal alternatives first)
+      //if (myActionInfos.Any(aInfo => !aInfo.Disabled && aInfo.Tries == 0)) {
+      //  return myActionInfos
+      //  .Select((aInfo, idx) => new { aInfo, idx })
+      //  .Where(p => !p.aInfo.Disabled)
+      //  .Where(p => p.aInfo.Tries == 0)
+      //  .SelectRandom(random).idx;
+      //}
+
+      var w = from aInfo in myActionInfos
+              select Math.Exp(beta * aInfo.Value);
+
+      var bestAction = Enumerable.Range(0, myActionInfos.Count()).SampleProportional(random, w);
+      Debug.Assert(bestAction >= 0);
+      return bestAction;
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+
+    public override string ToString() {
+      return string.Format("BoltzmannExplorationPolicy({0:F2})", beta);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ChernoffIntervalEstimationPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ChernoffIntervalEstimationPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ChernoffIntervalEstimationPolicy.cs	(revision 12290)
@@ -0,0 +1,63 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  /* see: Streeter and Smith: A simple distribution-free approach to the max k-armed bandit problem, Proceedings  of the 12th
+International Conference, CP 2006, Nantes, France, September 25-29, 2006. pp 560-574 */
+
+  public class ChernoffIntervalEstimationPolicy : IBanditPolicy {
+    private readonly double delta;
+
+    public ChernoffIntervalEstimationPolicy(double delta = 0.01) {
+      this.delta = delta;
+    }
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      Debug.Assert(actionInfos.Any());
+      // select best
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+      int k = myActionInfos.Count();
+      int totalTries = myActionInfos.Sum(a => a.Tries);
+      double bestQ = double.NegativeInfinity;
+      var bestActions = new List<int>();
+      var aIdx = -1;
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        if (aInfo.Tries == 0) {
+          q = double.PositiveInfinity;
+        } else {
+
+          var avgReward = aInfo.SumReward / aInfo.Tries;
+
+          // page 5 of "A simple distribution-free appraoch to the max k-armed bandit problem"
+          // var alpha = Math.Log(2 * totalTries * k / delta);
+          double alpha = Math.Log(2.0) + Math.Log(totalTries) + Math.Log(k) - Math.Log(delta);
+          // total tries is max tries in the original paper
+          q = avgReward + (alpha + Math.Sqrt(2 * aInfo.Tries * avgReward * alpha + alpha * alpha)) / aInfo.Tries;
+        }
+        if (q > bestQ) {
+          bestQ = q;
+          bestActions.Clear();
+          bestActions.Add(aIdx);
+        } else if (q.IsAlmost(bestQ)) {
+          bestActions.Add(aIdx);
+        }
+      }
+      Debug.Assert(bestActions.Any());
+      return bestActions.SelectRandom(random);
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+
+    public override string ToString() {
+      return string.Format("ChernoffIntervalEstimationPolicy({0:F2})", delta);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/EpsGreedyPolicy.cs	(revision 12290)
@@ -0,0 +1,63 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class EpsGreedyPolicy : IBanditPolicy {
+    private readonly double eps;
+    private readonly RandomPolicy randomPolicy;
+    private readonly string desc;
+
+
+    public EpsGreedyPolicy(double eps) : this(eps, string.Empty) { }
+
+    public EpsGreedyPolicy(double eps, string desc) {
+      this.eps = eps;
+      this.randomPolicy = new RandomPolicy();
+      this.desc = desc;
+    }
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      Debug.Assert(actionInfos.Any());
+      if (random.NextDouble() >= eps) { // eps == 0 should be equivalent to pure exploitation, eps == 1 is pure exploration
+        // select best
+        var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+        var bestActions = new List<int>();
+        double bestQ = double.NegativeInfinity;
+
+        int aIdx = -1;
+        foreach (var aInfo in myActionInfos) {
+          aIdx++;
+
+          var q = aInfo.Value;
+
+          if (q > bestQ) {
+            bestActions.Clear();
+            bestActions.Add(aIdx);
+            bestQ = q;
+          } else if (q.IsAlmost(bestQ)) {
+            bestActions.Add(aIdx);
+          }
+        }
+        Debug.Assert(bestActions.Any());
+        return bestActions.SelectRandom(random);
+      } else {
+        // select random
+        return randomPolicy.SelectAction(random, actionInfos);
+      }
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+
+
+    public override string ToString() {
+      return string.Format("EpsGreedyPolicy({0:F2},{1})", eps, desc);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/GaussianThompsonSamplingPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/GaussianThompsonSamplingPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/GaussianThompsonSamplingPolicy.cs	(revision 12290)
@@ -0,0 +1,86 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+
+  [Obsolete("Replaced by GenericThompsonSamplingPolicy(GaussianModel(0.5, 1.0, 0.1))")]
+  public class GaussianThompsonSamplingPolicy : IBanditPolicy {
+    private bool compatibility;
+
+    // assumes a Gaussian reward distribution with different means but the same variances for each action
+    // the prior for the mean is also Gaussian with the following parameters
+    private readonly double rewardVariance = 0.1; // we assume a known variance 
+
+    private readonly double priorMean = 0.5;
+    private readonly double priorVariance = 1;
+
+
+    public GaussianThompsonSamplingPolicy(bool compatibility = false) {
+      this.compatibility = compatibility;
+    }
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
+      int bestAction = -1;
+      double bestQ = double.NegativeInfinity;
+
+      int aIdx = -1;
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+
+        var tries = aInfo.Tries;
+        var sampleMean = aInfo.AvgReward;
+        var sampleVariance = aInfo.RewardVariance;
+
+        double theta;
+        if (compatibility) {
+          // old code used for old experiments (preserved because it performed very well)
+          if (tries < 2) return aIdx;
+          var mu = sampleMean;
+          var variance = sampleVariance;
+          var stdDev = Math.Sqrt(variance);
+          theta = Rand.RandNormal(random) * stdDev + mu;
+        } else {
+          // calculate posterior mean and variance (for mean reward)
+
+          // see Murphy 2007: Conjugate Bayesian analysis of the Gaussian distribution (http://www.cs.ubc.ca/~murphyk/Papers/bayesGauss.pdf)
+          var posteriorVariance = 1.0 / (tries / rewardVariance + 1.0 / priorVariance);
+          var posteriorMean = posteriorVariance * (priorMean / priorVariance + tries * sampleMean / rewardVariance);
+
+          // sample a mean from the posterior 
+          theta = Rand.RandNormal(random) * Math.Sqrt(posteriorVariance) + posteriorMean;
+
+          // theta already represents the expected reward value => nothing else to do
+        }
+
+        // very unlikely to be the same (and we don't care)
+        if (theta > bestQ) {
+          bestQ = theta;
+          bestAction = aIdx;
+        }
+      }
+      Debug.Assert(bestAction > -1);
+      return bestAction;
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new MeanAndVariancePolicyActionInfo();
+    }
+
+
+    //public override void UpdateReward(int action, double reward) {
+    //  Debug.Assert(Actions.Contains(action));
+    //  tries[action]++;
+    //  var delta = reward - sampleMean[action];
+    //  sampleMean[action] += delta / tries[action];
+    //  sampleM2[action] += sampleM2[action] + delta * (reward - sampleMean[action]);
+    //}
+
+    public override string ToString() {
+      return "GaussianThompsonSamplingPolicy";
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/GenericThompsonSamplingPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/GenericThompsonSamplingPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/GenericThompsonSamplingPolicy.cs	(revision 12290)
@@ -0,0 +1,43 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class GenericThompsonSamplingPolicy : IBanditPolicy {
+    private readonly IModel model;
+
+    public GenericThompsonSamplingPolicy(IModel model) {
+      this.model = model;
+    }
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<ModelPolicyActionInfo>();
+      int bestAction = -1;
+      double bestQ = double.NegativeInfinity;
+      var aIdx = -1;
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        var q = aInfo.SampleExpectedReward(random);
+        // very unlikely to be equal and we don't care
+        if (q > bestQ) {
+          bestQ = q;
+          bestAction = aIdx;
+        }
+      }
+      Debug.Assert(bestAction > -1);
+      return bestAction;
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new ModelPolicyActionInfo((IModel)model.Clone());
+    }
+
+    public override string ToString() {
+      return string.Format("GenericThompsonSamplingPolicy({0})", model);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ModifiedUCTPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ModifiedUCTPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ModifiedUCTPolicy.cs	(revision 12290)
@@ -0,0 +1,55 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  /* modified UCT: see Coquelin & Munos, Bandit Algorithms for Tree Search, UAI 2007 */
+  public class ModifiedUCTPolicy : IBanditPolicy {
+    private readonly double c;
+
+    public ModifiedUCTPolicy(double c = 1.0) {
+      this.c = c;
+    }
+
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+      double bestQ = double.NegativeInfinity;
+      int totalTries = myActionInfos.Sum(a => a.Tries);
+
+      int aIdx = -1;
+      var bestActions = new List<int>();
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        if (aInfo.Tries == 0) {
+          q = double.PositiveInfinity;
+        } else {
+          q = aInfo.SumReward / aInfo.Tries + c * Math.Sqrt(Math.Sqrt(totalTries) / aInfo.Tries); // modification in comparison to original UCT
+        }
+        if (q > bestQ) {
+          bestActions.Clear();
+          bestQ = q;
+          bestActions.Add(aIdx);
+        } else if (q.IsAlmost(bestQ)) {
+          bestActions.Add(aIdx);
+        }
+
+      }
+      Debug.Assert(bestActions.Any());
+      return bestActions.SelectRandom(random);
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+
+    public override string ToString() {
+      return string.Format("ModifiedUCTPolicy({0:F2})", c);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/RandomPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/RandomPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/RandomPolicy.cs	(revision 12290)
@@ -0,0 +1,26 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class RandomPolicy : IBanditPolicy {
+
+    public override string ToString() {
+      return "RandomPolicy";
+    }
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      return actionInfos
+        .Select((aInfo, idx) => Tuple.Create(aInfo, idx))
+        .SelectRandom(random).Item2;
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ThresholdAscentPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ThresholdAscentPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/ThresholdAscentPolicy.cs	(revision 12290)
@@ -0,0 +1,131 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  /* see: Streeter and Smith: A simple distribution-free approach to the max k-armed bandit problem, Proceedings  of the 12th
+ International Conference, CP 2006, Nantes, France, September 25-29, 2006. pp 560-574 */
+
+  public class ThresholdAscentPolicy : IBanditPolicy {
+    public const int numBins = 101;
+    public const double binSize = 1.0 / (numBins - 1);
+
+    private class ThresholdAscentActionInfo : IBanditPolicyActionInfo {
+
+      // for each arm store the number of observed rewards for each bin of size delta
+      // for delta = 0.01 we have 101 bins
+      // the first bin is freq of rewards  >= 0 // all
+      // the second bin is freq of rewards > 0
+      // the third bin is freq of rewards > 0.01
+      // the last bin is for rewards > 0.99
+      // 
+      // (also see RewardBin function)
+      public int[] rewardHistogram = new int[numBins];    // for performance reasons we store cumulative counts (freq of rewards > lower threshold)
+      public int Tries { get; private set; }
+      public int thresholdBin = 1;
+
+      public double Value {
+        get {
+          if (Tries == 0.0) return double.PositiveInfinity;
+          return rewardHistogram[thresholdBin] / (double)Tries;
+        }
+      }
+
+      public void UpdateReward(double reward) {
+        Tries++;
+        for (var idx = thresholdBin; idx <= RewardBin(reward); idx++)
+          rewardHistogram[idx]++;
+      }
+
+      public void Reset() {
+        Tries = 0;
+        thresholdBin = 1;
+        Array.Clear(rewardHistogram, 0, rewardHistogram.Length);
+      }
+
+      // maps a reward value to it's bin
+      private static int RewardBin(double reward) {
+        Debug.Assert(reward >= 0 && reward <= 1.0);
+        // reward = 0 => 0
+        // ]0.00 .. 0.01] => 1
+        // ]0.01 .. 0.02] => 2
+        // ...
+        // ]0.99 .. 1.00] => 100
+        if (reward <= 0) return 0;
+        return (int)Math.Ceiling((reward / binSize));
+      }
+    }
+
+    private readonly int s;
+    private readonly double delta;
+
+    public ThresholdAscentPolicy(int s = 100, double delta = 0.05) {
+      this.s = s;
+      this.delta = delta;
+    }
+
+    private double U(double mu, double totalTries, int n, int k) {
+      //var alpha = Math.Log(2.0 * totalTries * k / delta);
+      double alpha = Math.Log(2) + Math.Log(totalTries) + Math.Log(k) - Math.Log(delta);
+      return mu + (alpha + Math.Sqrt(2 * n * mu * alpha + alpha * alpha)) / n;
+    }
+
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      Debug.Assert(actionInfos.Any());
+      var myActionInfos = actionInfos.OfType<ThresholdAscentActionInfo>();
+      UpdateThreshold(myActionInfos);
+
+      var bestActions = new List<int>();
+      double bestQ = double.NegativeInfinity;
+      int k = myActionInfos.Count();
+      var totalTries = myActionInfos.Sum(a => a.Tries);
+      int aIdx = -1;
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        if (aInfo.Tries == 0) {
+          q = double.PositiveInfinity;
+        } else {
+          double mu = aInfo.Value; // probability of rewards > T 
+          q = U(mu, totalTries, aInfo.Tries, k); // totalTries is max iterations in original paper
+        }
+        if (q > bestQ) {
+          bestQ = q;
+          bestActions.Clear();
+          bestActions.Add(aIdx);
+        } else if (q.IsAlmost(bestQ)) {
+          bestActions.Add(aIdx);
+        }
+      }
+      Debug.Assert(bestActions.Any());
+      return bestActions.SelectRandom(random);
+    }
+
+
+    private void UpdateThreshold(IEnumerable<ThresholdAscentActionInfo> actionInfos) {
+      var thresholdBin = 1; // first bin to check is bin idx 1 == freq of rewards > 0
+      while (thresholdBin < (numBins - 1) && actionInfos.Sum(a => a.rewardHistogram[thresholdBin]) >= s) {
+        thresholdBin++;
+        // Console.WriteLine("New threshold {0:F2}", T);
+      }
+      foreach (var aInfo in actionInfos) {
+        aInfo.thresholdBin = thresholdBin;
+      }
+    }
+
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new ThresholdAscentActionInfo();
+    }
+
+    public override string ToString() {
+      return string.Format("ThresholdAscentPolicy({0},{1:F2})", s, delta);
+    }
+
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1Policy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1Policy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1Policy.cs	(revision 12290)
@@ -0,0 +1,47 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  // policy for k-armed bandit (see Auer et al. 2002)
+  public class UCB1Policy : IBanditPolicy {
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+      double bestQ = double.NegativeInfinity;
+      int totalTries = myActionInfos.Sum(a => a.Tries);
+
+      var bestActions = new List<int>();
+      int aIdx = -1;
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        if (aInfo.Tries == 0) {
+          q = double.PositiveInfinity;
+        } else {
+
+          q = aInfo.SumReward / aInfo.Tries + 0.5 * Math.Sqrt((2 * Math.Log(totalTries)) / aInfo.Tries);
+        }
+        if (q > bestQ) {
+          bestQ = q;
+          bestActions.Clear();
+          bestActions.Add(aIdx);
+        } else if (q.IsAlmost(bestQ)) {
+          bestActions.Add(aIdx);
+        }
+      }
+      Debug.Assert(bestActions.Any());
+      return bestActions.SelectRandom(random);
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+    public override string ToString() {
+      return "UCB1Policy";
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1TunedPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1TunedPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCB1TunedPolicy.cs	(revision 12290)
@@ -0,0 +1,60 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  // policy for k-armed bandit (see Auer et al. 2002)
+  public class UCB1TunedPolicy : IBanditPolicy {
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
+
+      int totalTries = myActionInfos.Sum(a => a.Tries);
+
+      int aIdx = -1;
+      double bestQ = double.NegativeInfinity;
+      var bestActions = new List<int>();
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        if (aInfo.Tries == 0) {
+          q = double.PositiveInfinity;
+        } else {
+          var sumReward = aInfo.SumReward;
+          var tries = aInfo.Tries;
+
+          var avgReward = sumReward / tries;
+          q = avgReward + Math.Sqrt((Math.Log(totalTries) / tries) * Math.Min(1.0 / 4, V(aInfo, totalTries)));
+          // 1/4 is upper bound of bernoulli distributed variable
+        }
+        if (q > bestQ) {
+          bestQ = q;
+          bestActions.Clear();
+          bestActions.Add(aIdx);
+        } else if (q.IsAlmost(bestQ)) {
+          bestActions.Add(aIdx);
+        }
+      }
+      Debug.Assert(bestActions.Any());
+
+      return bestActions.SelectRandom(random);
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new MeanAndVariancePolicyActionInfo();
+    }
+
+    private double V(MeanAndVariancePolicyActionInfo actionInfo, int totalTries) {
+      var s = actionInfo.Tries;
+      return actionInfo.RewardVariance + Math.Sqrt(2 * Math.Log(totalTries) / s);
+    }
+
+    public override string ToString() {
+      return "UCB1TunedPolicy";
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCBNormalPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCBNormalPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCBNormalPolicy.cs	(revision 12290)
@@ -0,0 +1,50 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  public class UCBNormalPolicy : IBanditPolicy {
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<MeanAndVariancePolicyActionInfo>();
+      int totalTries = myActionInfos.Sum(a => a.Tries);
+      double bestQ = double.NegativeInfinity;
+      int aIdx = -1;
+      var bestActions = new List<int>();
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        if (totalTries <= 1 || aInfo.Tries <= 1 || aInfo.Tries <= Math.Ceiling(8 * Math.Log(totalTries))) {
+          q = double.PositiveInfinity;
+        } else {
+          var tries = aInfo.Tries;
+          var avgReward = aInfo.AvgReward;
+          var rewardVariance = aInfo.RewardVariance;
+          var estVariance = 16.0 * rewardVariance * (Math.Log(totalTries - 1) / tries);
+          q = avgReward + Math.Sqrt(estVariance);
+        }
+        if (q > bestQ) {
+          bestQ = q;
+          bestActions.Clear();
+          bestActions.Add(aIdx);
+        } else if (q.IsAlmost(bestQ)) {
+          bestActions.Add(aIdx);
+        }
+      }
+      Debug.Assert(bestActions.Any());
+      return bestActions.SelectRandom(random);
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new MeanAndVariancePolicyActionInfo();
+    }
+
+    public override string ToString() {
+      return "UCBNormalPolicy";
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCTPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCTPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Policies/UCTPolicy.cs	(revision 12290)
@@ -0,0 +1,55 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.BanditPolicies {
+  /* Kocsis et al. Bandit based Monte-Carlo Planning */
+  public class UCTPolicy : IBanditPolicy {
+    private readonly double c;
+
+    public UCTPolicy(double c = 1.0) {
+      this.c = c;
+    }
+
+
+    public int SelectAction(Random random, IEnumerable<IBanditPolicyActionInfo> actionInfos) {
+      var myActionInfos = actionInfos.OfType<DefaultPolicyActionInfo>();
+      double bestQ = double.NegativeInfinity;
+      int totalTries = myActionInfos.Sum(a => a.Tries);
+
+      int aIdx = -1;
+      var bestActions = new List<int>();
+      foreach (var aInfo in myActionInfos) {
+        aIdx++;
+        double q;
+        if (aInfo.Tries == 0) {
+          q = double.PositiveInfinity;
+        } else {
+          q = aInfo.SumReward / aInfo.Tries + 2.0 * c * Math.Sqrt(Math.Log(totalTries) / aInfo.Tries);
+        }
+        if (q > bestQ) {
+          bestActions.Clear();
+          bestQ = q;
+          bestActions.Add(aIdx);
+        } else if (q.IsAlmost(bestQ)) {
+          bestActions.Add(aIdx);
+        }
+
+      }
+      Debug.Assert(bestActions.Any());
+      return bestActions.SelectRandom(random);
+    }
+
+    public IBanditPolicyActionInfo CreateActionInfo() {
+      return new DefaultPolicyActionInfo();
+    }
+
+    public override string ToString() {
+      return string.Format("UCTPolicy({0:F2})", c);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.Bandits/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Algorithms.Bandits")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("FH-Hagenberg")]
+[assembly: AssemblyProduct("HeuristicLab.Algorithms.Bandits")]
+[assembly: AssemblyCopyright("Copyright © FH-Hagenberg 2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("879afae5-fa8a-4b3a-90c8-f8718530721a")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprEvaluator.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprEvaluator.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprEvaluator.cs	(revision 12290)
@@ -0,0 +1,64 @@
+﻿using System;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Operators;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  [NonDiscoverableType]
+  [StorableClass]
+  [Item("GenericSymbExprEvaluator", "Evaluator for grammatical optimization problems (using a symbolic expression tree encoding).")]
+  public class GenericSymbExprEvaluator : SingleSuccessorOperator, IGenericSymbExprEvaluator {
+    public event Action<string, double> SolutionEvaluated;
+
+    public ILookupParameter<DoubleValue> QualityParameter {
+      get { return (ILookupParameter<DoubleValue>)Parameters["Quality"]; }
+    }
+    public ILookupParameter<ISymbolicExpressionTree> SymbolicExpressionTreeParameter {
+      get { return (ILookupParameter<ISymbolicExpressionTree>)Parameters["SymbolicExpressionTree"]; }
+    }
+
+    // cannot save these (eval won't work when loaded from file
+    private Func<string, double> evalFunc;
+    private Func<ISymbolicExpressionTree, string> toStringFunc;
+
+    [StorableConstructor]
+    private GenericSymbExprEvaluator(bool deserializing) : base(deserializing) { }
+    public GenericSymbExprEvaluator(GenericSymbExprEvaluator original, Cloner cloner)
+      : base(original, cloner) {
+      this.evalFunc = original.evalFunc;
+    }
+
+    public GenericSymbExprEvaluator(Func<ISymbolicExpressionTree, string> toStringFunc, Func<string, double> evalFunc)
+      : base() {
+      Parameters.Add(new LookupParameter<DoubleValue>("Quality"));
+      Parameters.Add(new LookupParameter<ISymbolicExpressionTree>("SymbolicExpressionTree"));
+      this.evalFunc = evalFunc;
+      this.toStringFunc = toStringFunc;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GenericSymbExprEvaluator(this, cloner);
+    }
+
+    public override IOperation Apply() {
+      var tree = SymbolicExpressionTreeParameter.ActualValue;
+      var sentence = toStringFunc(tree);
+      var q = evalFunc(sentence);
+      QualityParameter.ActualValue = new DoubleValue(q);
+
+      RaiseSolutionEvaluated(sentence, q);
+
+      return base.Apply();
+    }
+
+    private void RaiseSolutionEvaluated(string sentence, double quality) {
+      var handler = SolutionEvaluated;
+      if (handler != null) handler(sentence, quality);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprGrammar.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprGrammar.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprGrammar.cs	(revision 12290)
@@ -0,0 +1,104 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  [NonDiscoverableType]
+  [StorableClass]
+  [Item("GenericSymbExprGrammar", "Represents a general grammar for grammatical optimization problems.")]
+  public class GenericSymbExprGrammar : SymbolicExpressionGrammar {
+    [StorableClass]
+    [Item("GenericSymbol", "")]
+    public sealed class GenericSymbol : Symbol {
+      private readonly int minimumArity = 1;
+      private readonly int maximumArity = byte.MaxValue;
+
+      public override int MinimumArity { get { return minimumArity; } }
+      public override int MaximumArity { get { return maximumArity; } }
+
+      [StorableConstructor]
+      private GenericSymbol(bool deserializing) : base(deserializing) { }
+      private GenericSymbol(GenericSymbol original, Cloner cloner) : base(original, cloner) { }
+      public override IDeepCloneable Clone(Cloner cloner) {
+        return new GenericSymbol(this, cloner);
+      }
+      public GenericSymbol(char s, int minArity, int maxArity)
+        : base(s.ToString(), string.Empty) {
+        this.minimumArity = minArity;
+        this.maximumArity = maxArity;
+      }
+    }
+
+    [StorableConstructor]
+    public GenericSymbExprGrammar(bool deserializing) : base(deserializing) { }
+
+    public GenericSymbExprGrammar(SymbolicExpressionGrammar original, Cloner cloner) : base(original, cloner) { }
+
+    public GenericSymbExprGrammar(IGrammar grammar)
+      : base(ItemAttribute.GetName(typeof(GenericSymbExprGrammar)), ItemAttribute.GetDescription(typeof(GenericSymbExprGrammar))) {
+
+      var ntSymbs = new List<GenericSymbol>();
+      var tSymbs = new List<GenericSymbol>();
+      // for terminal and nonterminal symbol in the grammar create a symbol class
+      // for non-terminals set the arity and the allowed children
+      // for terminals set arity to 0 and allowed children to empty
+      foreach (var nt in grammar.NonTerminalSymbols) {
+        var ntSymb = CreateSymbol(grammar, nt);
+        AddSymbol(ntSymb);
+        SetSubtreeCount(ntSymb, ntSymb.MinimumArity, ntSymb.MaximumArity);
+        ntSymbs.Add(ntSymb);
+      }
+      foreach (var t in grammar.TerminalSymbols) {
+        var tSymb = new GenericSymbol(t, 0, 0);
+        AddSymbol(tSymb);
+        SetSubtreeCount(tSymb, 0, 0);
+        tSymbs.Add(tSymb);
+      }
+
+      Func<char, GenericSymbol> findSymb = (char s) => {
+        var selectedSymb = ntSymbs.SingleOrDefault(symb => symb.Name == s.ToString());
+        if (selectedSymb != null) return selectedSymb;
+        return tSymbs.SingleOrDefault(symb => symb.Name == s.ToString());
+      };
+
+      // set the sentence symbol as the allowed start symbol
+      AddAllowedChildSymbol(StartSymbol, findSymb(grammar.SentenceSymbol));
+
+      foreach (var nt in ntSymbs) {
+        // either all alts are only a single symbol 
+        // or all alts only use a single symbol (see assertion in CreateSymbol)
+        var alts = grammar.GetAlternatives(nt.Name[0]); // all symbols only have a single character name
+        Debug.Assert(alts.All(alt => alt.Length == 1 || alt.Distinct().Count() == 1));
+
+        foreach (var alt in alts) {
+          Debug.Assert(alt.All(ch => ch == alt[0]));
+          AddAllowedChildSymbol(nt, findSymb(alt.First()));
+        }
+      }
+    }
+
+    private GenericSymbol CreateSymbol(IGrammar g, char s) {
+
+      var minArity = g.GetAlternatives(s).Select(alt => alt.Length).Min();
+      var maxArity = g.GetAlternatives(s).Select(alt => alt.Length).Max();
+      // either the min and max arity is 1 or the alternatives all use the same symbol (hl doesn't know about dependencies between positions in the alternative
+      Debug.Assert(
+        (minArity == 1 && maxArity == 1) ||
+        g.GetAlternatives(s).Select(alt => alt.Distinct()).All(alt => alt.Count() == 1));
+      var symb = new GenericSymbol(s, minArity, maxArity);
+      return symb;
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GenericSymbExprGrammar(this, cloner);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/GenericSymbExprProblem.cs	(revision 12290)
@@ -0,0 +1,80 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Common;
+using HeuristicLab.Core;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+using HeuristicLab.Parameters;
+using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
+using HeuristicLab.PluginInfrastructure;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  [NonDiscoverableType]
+  [StorableClass]
+  [Item("GenericSymbExprProblem", "Represents grammatical optimization problems (using a symbolic expression tree encoding).")]
+  public class GenericSymbExprProblem : SingleObjectiveHeuristicOptimizationProblem<IGenericSymbExprEvaluator, ISymbolicExpressionTreeCreator> {
+    #region string consts
+    private const string SymbolicExpressionTreeGrammarParameterName = "SymbolicExpressionTreeGrammar";
+    private const string MaximumSymbolicExpressionTreeDepthParameterName = "MaximumSymbolicExpressionTreeDepth";
+    private const string MaximumSymbolicExpressionTreeLengthParameterName = "MaximumSymbolicExpressionTreeLength";
+
+    #endregion
+
+    #region parameter properties
+    public IValueParameter<ISymbolicExpressionTreeGrammar> SymbolicExpressionTreeGrammarParameter {
+      get { return (IValueParameter<ISymbolicExpressionTreeGrammar>)Parameters[SymbolicExpressionTreeGrammarParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MaximumSymbolicExpressionTreeDepthParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaximumSymbolicExpressionTreeDepthParameterName]; }
+    }
+    public IFixedValueParameter<IntValue> MaximumSymbolicExpressionTreeLengthParameter {
+      get { return (IFixedValueParameter<IntValue>)Parameters[MaximumSymbolicExpressionTreeLengthParameterName]; }
+    }
+    #endregion
+
+
+    #region properties
+
+    public ISymbolicExpressionTreeGrammar SymbolicExpressionTreeGrammar {
+      get { return SymbolicExpressionTreeGrammarParameter.Value; }
+      set { SymbolicExpressionTreeGrammarParameter.Value = value; }
+    }
+    public IntValue MaximumSymbolicExpressionTreeDepth {
+      get { return MaximumSymbolicExpressionTreeDepthParameter.Value; }
+    }
+    public IntValue MaximumSymbolicExpressionTreeLength {
+      get { return MaximumSymbolicExpressionTreeLengthParameter.Value; }
+    }
+    public new GenericSymbExprEvaluator Evaluator {
+      get { return (GenericSymbExprEvaluator)base.Evaluator; }
+    }
+
+    #endregion
+    [StorableConstructor]
+    public GenericSymbExprProblem(bool deserializing) : base(deserializing) { }
+
+    public GenericSymbExprProblem(GenericSymbExprProblem original, Cloner cloner) : base(original, cloner) { }
+
+    public GenericSymbExprProblem(ISymbolicExpressionTreeProblem problem)
+      : base(new GenericSymbExprEvaluator(problem.ConvertTreeToSentence, problem.Evaluate), new ProbabilisticTreeCreator()) {
+      var grammar = new GenericSymbExprGrammar(problem.TreeBasedGPGrammar);
+      Parameters.Add(new ValueParameter<ISymbolicExpressionGrammar>(SymbolicExpressionTreeGrammarParameterName, grammar));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaximumSymbolicExpressionTreeLengthParameterName, new IntValue(50)));
+      Parameters.Add(new FixedValueParameter<IntValue>(MaximumSymbolicExpressionTreeDepthParameterName, new IntValue(15)));
+
+      Maximization = new BoolValue(true);
+      BestKnownQuality = new DoubleValue(1.0);
+
+      Operators.AddRange(ApplicationManager.Manager.GetInstances<ISymbolicExpressionTreeOperator>());
+    }
+
+    public override IDeepCloneable Clone(Cloner cloner) {
+      return new GenericSymbExprProblem(this, cloner);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/HeuristicLab.Algorithms.GeneticProgramming.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/HeuristicLab.Algorithms.GeneticProgramming.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/HeuristicLab.Algorithms.GeneticProgramming.csproj	(revision 12290)
@@ -0,0 +1,129 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{14BEC23F-63FD-4954-B8AE-E2F4962E9B57}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Algorithms.GeneticProgramming</RootNamespace>
+    <AssemblyName>HeuristicLab.Algorithms.GeneticProgramming</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="HeuristicLab.Algorithms.GeneticAlgorithm-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Algorithms.GeneticAlgorithm-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Algorithms.OffspringSelectionGeneticAlgorithm-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Analysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Analysis-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Collections-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Collections-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Common-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Core-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Core-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Data-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Data-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Operators-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Operators-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Optimization-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Optimization-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.ParallelEngine-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.ParallelEngine-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Parameters-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Parameters-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Persistence-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Persistence-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.PluginInfrastructure-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.PluginInfrastructure-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Selection-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Selection-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.SequentialEngine-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.SequentialEngine-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Drawing" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="IGPSolver.cs" />
+    <Compile Include="GenericSymbExprEvaluator.cs" />
+    <Compile Include="GenericSymbExprGrammar.cs" />
+    <Compile Include="GenericSymbExprProblem.cs" />
+    <Compile Include="IGenericSymbExprEvaluator.cs" />
+    <Compile Include="OffspringSelectionGP.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="StandardGP.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.GrammaticalOptimization\HeuristicLab.Algorithms.GrammaticalOptimization.csproj">
+      <Project>{EEA07488-1A51-412A-A52C-53B754A628B3}</Project>
+      <Name>HeuristicLab.Algorithms.GrammaticalOptimization</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.csproj">
+      <Project>{CB9DCCF6-667E-4A13-B82D-DBD6B45A045E}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/IGPSolver.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/IGPSolver.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/IGPSolver.cs	(revision 12290)
@@ -0,0 +1,10 @@
+﻿using HeuristicLab.Algorithms.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GeneticProgramming {
+  public interface IGPSolver : ISolver {
+    int PopulationSize { get; set; }
+    double MutationRate { get; set; }
+    int MaxSolutionSize { get; set; }
+    int MaxSolutionDepth { get; set; }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/IGenericSymbExprEvaluator.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/IGenericSymbExprEvaluator.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/IGenericSymbExprEvaluator.cs	(revision 12290)
@@ -0,0 +1,9 @@
+﻿using HeuristicLab.Core;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Optimization;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public interface IGenericSymbExprEvaluator : ISingleObjectiveEvaluator, ISymbolicExpressionTreeOperator {
+    ILookupParameter<ISymbolicExpressionTree> SymbolicExpressionTreeParameter { get; } 
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/OffspringSelectionGP.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/OffspringSelectionGP.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/OffspringSelectionGP.cs	(revision 12290)
@@ -0,0 +1,106 @@
+﻿using System;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Analysis;
+using HeuristicLab.Common;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Persistence.Default.Xml;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GeneticProgramming {
+  public class OffspringSelectionGP : SolverBase, IGPSolver {
+    public int PopulationSize { get; set; }
+    public double MutationRate { get; set; }
+    public int MaxSolutionSize { get; set; }
+    public int MaxSolutionDepth { get; set; }
+    public double ComparisonFactorLowerBound { get; set; }
+
+    private readonly ISymbolicExpressionTreeProblem problem;
+    private readonly Random random;
+    private readonly bool saveAlg;
+
+    public OffspringSelectionGP(ISymbolicExpressionTreeProblem problem, Random random, bool saveAlg = false) {
+      this.problem = problem;
+      this.random = random;
+      // default parameter values
+      PopulationSize = 100;
+      MutationRate = 0.15;
+      MaxSolutionSize = 100;
+      MaxSolutionDepth = 17;
+      ComparisonFactorLowerBound = 1.0;
+      this.saveAlg = saveAlg;
+    }
+
+    public override void Run(int maxEvaluations) {
+      var hlProblem = new GenericSymbExprProblem(problem);
+      var onEvalLocker = new object();
+      hlProblem.MaximumSymbolicExpressionTreeLength.Value = MaxSolutionSize;
+      hlProblem.MaximumSymbolicExpressionTreeDepth.Value = MaxSolutionDepth;
+
+      using (var wh = new AutoResetEvent(false)) {
+        var osga = new OffspringSelectionGeneticAlgorithm.OffspringSelectionGeneticAlgorithm();
+        // osga.Engine = new ParallelEngine.ParallelEngine();
+        osga.Engine = new SequentialEngine.SequentialEngine();
+        osga.ExceptionOccurred += (sender, args) => { Console.WriteLine(args.Value.Message); wh.Set(); };
+        osga.Stopped += (sender, args) => { wh.Set(); };
+
+        int numEvals = 0;
+        hlProblem.Evaluator.SolutionEvaluated += (sentence, quality) => {
+          // raise solution evaluated event for each GP solution, don't scale quality to 0..1 
+          // need to synchronize in case we are using a parallel engine
+          lock (onEvalLocker) {
+            OnSolutionEvaluated(sentence, quality);
+
+            // stop when maxEvals has been reached
+            if (numEvals++ >= maxEvaluations) {
+              osga.Stop();
+            }
+          }
+        };
+
+
+        osga.Problem = hlProblem;
+        var mutator = (MultiSymbolicExpressionTreeManipulator)osga.MutatorParameter.ValidValues.Single(op => op.Name == "MultiSymbolicExpressionTreeManipulator");
+        foreach (var op in mutator.Operators) {
+          if (op.Name == "ChangeNodeTypeManipulation"
+            || op.Name == "ReplaceBranchManipulation") mutator.Operators.SetItemCheckedState(op, true);
+          else mutator.Operators.SetItemCheckedState(op, false);
+        }
+        osga.Mutator = mutator;
+        osga.Crossover = osga.CrossoverParameter.ValidValues.Single(op => op.Name == "SubtreeSwappingCrossover");
+        osga.Selector = osga.SelectorParameter.ValidValues.Single(op => op.Name == "GenderSpecificSelection");
+        var multiAnalzer = (MultiAnalyzer)osga.Analyzer;
+        multiAnalzer.Operators.Add(new BestSymbolicExpressionTreeAnalyzer());
+
+        osga.PopulationSize.Value = PopulationSize;
+        osga.MaximumGenerations.Value = 1000000; // some very large value (we stop based on evaluations)
+        osga.MaximumSelectionPressure.Value = 1000000;
+        osga.MaximumEvaluatedSolutions.Value = maxEvaluations;
+        osga.MutationProbability.Value = MutationRate;
+        osga.ComparisonFactorLowerBound.Value = 1.0;
+        osga.ComparisonFactorUpperBound.Value = ComparisonFactorLowerBound;
+        osga.SuccessRatio.Value = 1.0;
+
+        osga.SetSeedRandomly = new BoolValue(false);
+        osga.Seed = new IntValue(random.Next());
+
+        osga.Prepare();
+        osga.Start();
+
+        wh.Dispose();
+
+        if (saveAlg) {
+          var path = @"C:\Users\P24581\Desktop";
+          var fileName = string.Format("osgp-{0}{1:D2}{2:D2}{3:D2}{4:D2}.hl", DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, DateTime.Now.Hour, DateTime.Now.Minute);
+          var fullPath = Path.Combine(path, fileName);
+          HeuristicLab.Persistence.Core.ConfigurationService.Instance.LoadSettings();
+          XmlGenerator.Serialize(osga, fullPath, CompressionLevel.Fastest);
+        }
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Algorithms.GeneticProgramming")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("FH-Hagenberg")]
+[assembly: AssemblyProduct("HeuristicLab.Algorithms.GeneticProgramming")]
+[assembly: AssemblyCopyright("Copyright © FH-Hagenberg 2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("4ae636d8-8823-4839-8321-d331e2cd1335")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/StandardGP.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/StandardGP.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GeneticProgramming/StandardGP.cs	(revision 12290)
@@ -0,0 +1,103 @@
+﻿using System;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Threading;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Common;
+using HeuristicLab.Data;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Persistence.Default.Xml;
+using HeuristicLab.Problems.GrammaticalOptimization;
+using HeuristicLab.Selection;
+
+namespace HeuristicLab.Algorithms.GeneticProgramming {
+  public class StandardGP : SolverBase, IGPSolver {
+    public int PopulationSize { get; set; }
+    public double MutationRate { get; set; }
+    public int TournamentGroupSize { get; set; }
+    public int MaxSolutionSize { get; set; }
+    public int MaxSolutionDepth { get; set; }
+
+    private readonly ISymbolicExpressionTreeProblem problem;
+    private readonly Random random;
+    private readonly bool saveAlg;
+
+    public StandardGP(ISymbolicExpressionTreeProblem problem, Random random, bool saveAlg = false) {
+      this.problem = problem;
+      this.random = random;
+      // default parameter values
+      PopulationSize = 1000;
+      TournamentGroupSize = 4;
+      MutationRate = 0.15;
+      MaxSolutionSize = 100;
+      MaxSolutionDepth = 17;
+      this.saveAlg = saveAlg;
+    }
+
+    public override void Run(int maxEvaluations) {
+      var hlProblem = new GenericSymbExprProblem(problem);
+      var onEvalLocker = new object();
+      hlProblem.MaximumSymbolicExpressionTreeLength.Value = MaxSolutionSize;
+      hlProblem.MaximumSymbolicExpressionTreeDepth.Value = MaxSolutionDepth;
+
+
+      using (var wh = new AutoResetEvent(false)) {
+        var ga = new GeneticAlgorithm.GeneticAlgorithm();
+        ga.Engine = new ParallelEngine.ParallelEngine();
+        ga.ExceptionOccurred += (sender, args) => { Console.WriteLine(args.Value.Message); wh.Set(); };
+        ga.Stopped += (sender, args) => { wh.Set(); };
+
+        int numEvals = 0;
+        hlProblem.Evaluator.SolutionEvaluated += (sentence, quality) => {
+          // raise solution evaluated event for each GP solution, don't scale quality to 0..1 
+          // need to synchronize in case we are using a parallel engine
+          lock (onEvalLocker) {
+            OnSolutionEvaluated(sentence, quality);
+
+            // stop when maxEvals has been reached
+            if (numEvals++ >= maxEvaluations) {
+              ga.Stop();
+            }
+          }
+        };
+
+
+        ga.Problem = hlProblem;
+        //ga.Problem.SolutionCreatorParameter.ActualValue = new FullTreeCreator();
+        var mutator = (MultiSymbolicExpressionTreeManipulator)ga.MutatorParameter.ValidValues.Single(op => op.Name == "MultiSymbolicExpressionTreeManipulator");
+        foreach (var op in mutator.Operators) {
+          if (op.Name == "ChangeNodeTypeManipulation"
+            || op.Name == "ReplaceBranchManipulation") mutator.Operators.SetItemCheckedState(op, true);
+          else mutator.Operators.SetItemCheckedState(op, false);
+        }
+        ga.Mutator = mutator;
+        ga.Crossover = ga.CrossoverParameter.ValidValues.Single(op => op.Name == "SubtreeSwappingCrossover");
+        var selector = (TournamentSelector)ga.SelectorParameter.ValidValues.Single(op => op.Name == "TournamentSelector");
+        selector.GroupSizeParameter.Value = new IntValue(TournamentGroupSize);
+        ga.Selector = selector;
+
+        ga.PopulationSize.Value = PopulationSize;
+        ga.MaximumGenerations.Value = 1000000; // very large value (we stop in the evaluate handler)
+        ga.MutationProbability.Value = MutationRate;
+
+        ga.SetSeedRandomly = new BoolValue(false);
+        ga.Seed = new IntValue(random.Next());
+
+        ga.Prepare();
+        ga.Start();
+
+        wh.WaitOne();
+
+
+        if (saveAlg) {
+          var path = @"C:\Users\P24581\Desktop";
+          var fileName = string.Format("osgp-{0}{1:D2}{2:D2}{3:D2}{4:D2}.hl", DateTime.Now.Year, DateTime.Now.Month, DateTime.Now.Day, DateTime.Now.Hour, DateTime.Now.Minute);
+          var fullPath = Path.Combine(path, fileName);
+          HeuristicLab.Persistence.Core.ConfigurationService.Instance.LoadSettings();
+          XmlGenerator.Serialize(ga, fullPath, CompressionLevel.Fastest);
+        }
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/HeuristicLab.Algorithms.GrammaticalOptimization.csproj	(revision 12290)
@@ -0,0 +1,78 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{EEA07488-1A51-412A-A52C-53B754A628B3}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Algorithms.GrammaticalOptimization</RootNamespace>
+    <AssemblyName>HeuristicLab.Algorithms.GrammaticalOptimization</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ISequentialDecisionPolicy.cs" />
+    <Compile Include="SequentialDecisionPolicies\GenericPolicy.cs" />
+    <Compile Include="SequentialDecisionPolicies\GenericFunctionApproximationGrammarPolicy.cs" />
+    <Compile Include="SequentialDecisionPolicies\GenericGrammarPolicy.cs" />
+    <Compile Include="SequentialDecisionPolicies\GrammarPolicy.cs" />
+    <Compile Include="SequentialDecisionPolicies\IGrammarPolicy.cs" />
+    <Compile Include="SequentialDecisionPolicies\RandomPolicy.cs" />
+    <Compile Include="SolverBase.cs" />
+    <Compile Include="ISolver.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Solvers\ExhaustiveBreadthFirstSearch.cs" />
+    <Compile Include="Solvers\ExhaustiveDepthFirstSearch.cs" />
+    <Compile Include="Solvers\ExhaustiveRandomFirstSearch.cs" />
+    <Compile Include="Solvers\RandomSearch.cs" />
+    <Compile Include="Solvers\SequentialSearch.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.Bandits\HeuristicLab.Algorithms.Bandits.csproj">
+      <Project>{24408F7D-EE0F-4886-A08B-EC324D662E47}</Project>
+      <Name>HeuristicLab.Algorithms.Bandits</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.csproj">
+      <Project>{cb9dccf6-667e-4a13-b82d-dbd6b45a045e}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/ISequentialDecisionPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/ISequentialDecisionPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/ISequentialDecisionPolicy.cs	(revision 12290)
@@ -0,0 +1,25 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Dynamic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  // this interface represents a policy for episodic reinforcement learning (with afterstates)
+  // here we assume that a reward is only recieved at the end of the episode and the update is done only after an episode is complete
+  // we also assume that the policy can fail to select one of the followStates
+  public interface ISequentialDecisionPolicy<in TState> {
+    bool TrySelect(Random random, TState curState, IEnumerable<TState> afterStates, out int selectedStateIdx); // selectedState \in afterStates
+
+    // state-trajectory are the states of the episode, at the end we recieved the reward (only for the terminal state)
+    void UpdateReward(IEnumerable<TState> stateTrajectory, double reward);
+
+    void Reset(); // clears all internal state
+
+    // for introspection
+    double GetValue(TState state); 
+    int GetTries(TState state); 
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/ISolver.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/ISolver.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/ISolver.cs	(revision 12290)
@@ -0,0 +1,13 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.GrammaticalOptimization {
+  public interface ISolver {
+    event Action<string, double> FoundNewBestSolution;
+    event Action<string, double> SolutionEvaluated;
+    void Run(int maxEvaluations);
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Algorithms.GrammaticalOptimization")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("HeuristicLab.Algorithms.GrammaticalOptimization")]
+[assembly: AssemblyCopyright("Copyright © Microsoft 2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("074345f1-0e24-4ab7-8a24-ace6874d162b")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GenericFunctionApproximationGrammarPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GenericFunctionApproximationGrammarPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GenericFunctionApproximationGrammarPolicy.cs	(revision 12290)
@@ -0,0 +1,193 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.ExceptionServices;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.Bandits.GrammarPolicies {
+  public sealed class GenericFunctionApproximationGrammarPolicy : IGrammarPolicy {
+    private Dictionary<string, double> featureWeigths; // stores the necessary information for bandit policies for each state (=canonical phrase)
+    private Dictionary<string, int> featureTries;
+    private HashSet<string> done;
+    private readonly bool useCanonicalPhrases;
+    private readonly IProblem problem;
+
+
+
+    public GenericFunctionApproximationGrammarPolicy(IProblem problem, bool useCanonicalPhrases = false) {
+      this.useCanonicalPhrases = useCanonicalPhrases;
+      this.problem = problem;
+      this.featureWeigths = new Dictionary<string, double>();
+      this.featureTries = new Dictionary<string, int>();
+      this.done = new HashSet<string>();
+    }
+
+    private double[] activeAfterStates; // don't allocate each time
+    private int[] actionIndexMap; // don't allocate each time
+
+    public bool TrySelect(Random random, string curState, IEnumerable<string> afterStates, out int selectedStateIdx) {
+      // fail if all states are done (corresponding state infos are disabled)
+      if (afterStates.All(s => Done(s))) {
+        // fail because all follow states have already been visited => also disable the current state (if we can be sure that it has been fully explored)
+        MarkAsDone(curState);
+
+        selectedStateIdx = -1;
+        return false;
+      }
+
+      // determine active actions (not done yet) and create an array to map the selected index back to original actions
+      if (activeAfterStates == null || activeAfterStates.Length < afterStates.Count()) {
+        activeAfterStates = new double[afterStates.Count()];
+        actionIndexMap = new int[afterStates.Count()];
+      }
+      var maxIdx = 0; int originalIdx = 0;
+      foreach (var afterState in afterStates) {
+        if (!Done(afterState)) {
+          activeAfterStates[maxIdx] = 0.0;
+          actionIndexMap[maxIdx] = originalIdx;
+
+
+          activeAfterStates[maxIdx] = GetValue(afterState);
+
+          maxIdx++;
+        }
+        originalIdx++;
+      }
+      
+      
+      // TODO: policy should be a parameter of the function approximation policy
+      if (random.NextDouble() < 0.2) {
+        selectedStateIdx = actionIndexMap[random.Next(maxIdx)];
+      } else {
+        // find max
+        var bestQ = double.NegativeInfinity;
+        var bestIdxs = new List<int>();
+        for (int i = 0; i < maxIdx; i++) {
+          if (activeAfterStates[i] > bestQ) {
+            bestIdxs.Clear();
+            bestIdxs.Add(i);
+            bestQ = activeAfterStates[i];
+          } else if (activeAfterStates[i].IsAlmost(bestQ)) {
+            bestIdxs.Add(i);
+          }
+        }
+        selectedStateIdx = actionIndexMap[bestIdxs[random.Next(bestIdxs.Count)]];
+      }
+      
+      return true;
+    }
+
+
+    public void UpdateReward(IEnumerable<string> stateTrajectory, double reward) {
+      foreach (var state in stateTrajectory) {
+        UpdateWeights(state, reward);
+
+        // only the last state can be terminal
+        if (problem.Grammar.IsTerminal(state)) {
+          MarkAsDone(state);
+        }
+      }
+    }
+
+
+    private IEnumerable<KeyValuePair<string, double>> Values {
+      get { return featureWeigths.OrderByDescending(p => p.Value); }
+    }
+
+    public void Reset() {
+      featureWeigths.Clear();
+      done.Clear();
+    }
+
+    public int GetTries(string state) {
+      return 0;
+    }
+
+    public int GetFeatureTries(string featureId) {
+      int t;
+      if (featureTries.TryGetValue(featureId, out t)) {
+        return t;
+      } else return 0;
+    }
+
+    public double GetValue(string state) {
+      return problem.GetFeatures(state).Sum(feature => GetWeight(feature));
+    }
+
+    private double GetWeight(Feature feature) {
+      double w;
+      if (featureWeigths.TryGetValue(feature.Id, out w)) return w * feature.Value;
+      else return 0.0;
+    }
+    private void UpdateWeights(string state, double reward) {
+      double delta = reward - GetValue(state);
+      foreach (var feature in problem.GetFeatures(state)) {
+        //featureTries[feature.Id] = GetFeatureTries(feature.Id) + 1;
+        //Debug.Assert(GetFeatureTries(feature.Id) >= 1);
+        //double alpha = 1.0 / GetFeatureTries(feature.Id);
+        //alpha = Math.Max(alpha, 0.001);
+        
+        // simple setting of constant alpha = 0.01 works very well for poly-10 (100% success rate for 20 runs within 40000 evaluations))
+        var alpha = 0.01;
+
+        double w;
+        if (!featureWeigths.TryGetValue(feature.Id, out w)) {
+          featureWeigths[feature.Id] = alpha * delta * feature.Value;
+        } else {
+          featureWeigths[feature.Id] += alpha * delta * feature.Value;
+        }
+      }
+    }
+
+
+
+    // the canonical states for the value function (banditInfos) and the done set must be distinguished
+    // sequences of different length could have the same canonical representation and can have the same value (banditInfo)
+    // however, if the canonical representation of a state is shorter than we must not mark the canonical state as done when all possible derivations from the initial state have been explored
+    // eg. in the ant problem the canonical representation for ...lllA is ...rA
+    // even though all possible derivations (of limited length) of lllA have been visited we must not mark the state rA as done
+    private void MarkAsDone(string state) {
+      var s = CanonicalState(state);
+      // when the lengths of the canonical string and the original string are the same we also disable the actions
+      // always disable terminals
+      Debug.Assert(s.Length <= state.Length);
+      if (s.Length == state.Length || problem.Grammar.IsTerminal(state)) {
+        Debug.Assert(!done.Contains(s));
+        done.Add(s);
+      } else {
+        // for non-terminals where the canonical string is shorter than the original string we can only disable the canonical representation for all states in the same level
+        Debug.Assert(!done.Contains(s + state.Length));
+        done.Add(s + state.Length); // encode the original length of the state, states in the same level of the tree are treated as equivalent
+      }
+    }
+
+    // symmetric to MarkDone
+    private bool Done(string state) {
+      var s = CanonicalState(state);
+      if (s.Length == state.Length || problem.Grammar.IsTerminal(state)) {
+        return done.Contains(s);
+      } else {
+        // it is not necessary to visit states if the canonical representation has already been fully explored
+        if (done.Contains(s)) return true;
+        if (done.Contains(s + state.Length)) return true;
+        for (int i = 1; i < state.Length; i++) {
+          if (done.Contains(s + i)) return true;
+        }
+        return false;
+      }
+    }
+
+    private string CanonicalState(string state) {
+      if (useCanonicalPhrases) {
+        return problem.CanonicalRepresentation(state);
+      } else
+        return state;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GenericGrammarPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GenericGrammarPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GenericGrammarPolicy.cs	(revision 12290)
@@ -0,0 +1,146 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.Bandits.GrammarPolicies {
+  // this represents grammar policies that use one of the available bandit policies for state selection
+  // any bandit policy can be used to select actions for states
+  // a separate datastructure is used to store visited states and to prevent revisiting of states
+  public sealed class GenericGrammarPolicy : IGrammarPolicy {
+    private Dictionary<string, IBanditPolicyActionInfo> stateInfo; // stores the necessary information for bandit policies for each state (=canonical phrase)
+    private HashSet<string> done;
+    private readonly bool useCanonicalPhrases;
+    private readonly IProblem problem;
+    private readonly IBanditPolicy banditPolicy;
+
+    public GenericGrammarPolicy(IProblem problem, IBanditPolicy banditPolicy, bool useCanonicalPhrases = false) {
+      this.useCanonicalPhrases = useCanonicalPhrases;
+      this.problem = problem;
+      this.banditPolicy = banditPolicy;
+      this.stateInfo = new Dictionary<string, IBanditPolicyActionInfo>();
+      this.done = new HashSet<string>();
+    }
+
+    private IBanditPolicyActionInfo[] activeAfterStates; // don't allocate each time
+    private int[] actionIndexMap; // don't allocate each time
+
+    public bool TrySelect(Random random, string curState, IEnumerable<string> afterStates, out int selectedStateIdx) {
+      // fail if all states are done (corresponding state infos are disabled)
+      if (afterStates.All(s => Done(s))) {
+        // fail because all follow states have already been visited => also disable the current state (if we can be sure that it has been fully explored)
+        MarkAsDone(curState);
+
+        selectedStateIdx = -1;
+        return false;
+      }
+
+      // determine active actions (not done yet) and create an array to map the selected index back to original actions
+      if (activeAfterStates == null || activeAfterStates.Length < afterStates.Count()) {
+        activeAfterStates = new IBanditPolicyActionInfo[afterStates.Count()];
+        actionIndexMap = new int[afterStates.Count()];
+      }
+      var idx = 0; int originalIdx = 0;
+      foreach (var afterState in afterStates) {
+        if (!Done(afterState)) {
+          activeAfterStates[idx] = GetStateInfo(afterState);
+          actionIndexMap[idx] = originalIdx;
+          idx++;
+        }
+        originalIdx++;
+      }
+
+      selectedStateIdx = actionIndexMap[banditPolicy.SelectAction(random, activeAfterStates.Take(idx))];
+
+      return true;
+    }
+
+
+
+    private IBanditPolicyActionInfo GetStateInfo(string state) {
+      var s = CanonicalState(state);
+      IBanditPolicyActionInfo info;
+      if (!stateInfo.TryGetValue(s, out info)) {
+        info = banditPolicy.CreateActionInfo();
+        stateInfo[s] = info;
+      }
+      return info;
+    }
+
+    public void UpdateReward(IEnumerable<string> stateTrajectory, double reward) {
+      foreach (var state in stateTrajectory) {
+        GetStateInfo(state).UpdateReward(reward);
+
+        // only the last state can be terminal
+        if (problem.Grammar.IsTerminal(state)) {
+          MarkAsDone(state);
+        }
+      }
+    }
+
+
+    public void Reset() {
+      stateInfo.Clear();
+      done.Clear();
+    }
+
+    public int GetTries(string state) {
+      var s = CanonicalState(state);
+      if (stateInfo.ContainsKey(s)) return stateInfo[s].Tries;
+      else return 0;
+    }
+
+    public double GetValue(string state) {
+      var s = CanonicalState(state);
+      if (stateInfo.ContainsKey(s)) return stateInfo[s].Value;
+      else return 0.0; // TODO: check alternatives
+    }
+
+    // the canonical states for the value function (banditInfos) and the done set must be distinguished
+    // sequences of different length could have the same canonical representation and can have the same value (banditInfo)
+    // however, if the canonical representation of a state is shorter then we must not mark the canonical state as done when all possible derivations from the initial state have been explored
+    // eg. in the ant problem the canonical representation for ...lllA is ...rA
+    // even though all possible derivations (of limited length) of lllA have been visited we must not mark the state rA as done
+    private void MarkAsDone(string state) {
+      var s = CanonicalState(state);
+      // when the lengths of the canonical string and the original string are the same we also disable the actions
+      // always disable terminals
+      Debug.Assert(s.Length <= state.Length);
+      if (s.Length == state.Length || problem.Grammar.IsTerminal(state)) {
+        Debug.Assert(!done.Contains(s));
+        done.Add(s);
+      } else {
+        // for non-terminals where the canonical string is shorter than the original string we can only disable the canonical representation for all states in the same level
+        Debug.Assert(!done.Contains(s + state.Length));
+        done.Add(s + state.Length); // encode the original length of the state, states in the same level of the tree are treated as equivalent
+      }
+    }
+
+    // symmetric to MarkDone
+    private bool Done(string state) {
+      var s = CanonicalState(state);
+      if (s.Length == state.Length || problem.Grammar.IsTerminal(state)) {
+        return done.Contains(s);
+      } else {
+        // it is not necessary to visit states if the canonical representation has already been fully explored
+        if (done.Contains(s)) return true;
+        if (done.Contains(s + state.Length)) return true;
+        for (int i = 1; i < state.Length; i++) {
+          if (done.Contains(s + i)) return true;
+        }
+        return false;
+      }
+    }
+
+    private string CanonicalState(string state) {
+      if (useCanonicalPhrases) {
+        return problem.CanonicalRepresentation(state);
+      } else
+        return state;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GrammarPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GrammarPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/GrammarPolicy.cs	(revision 12290)
@@ -0,0 +1,76 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.Bandits.GrammarPolicies {
+  // stores: tries, avg reward and max reward for each state (base class for RandomPolicy and TDPolicy
+  public abstract class GrammarPolicy : IGrammarPolicy {
+    protected Dictionary<string, double> avgReward;
+    protected Dictionary<string, int> tries;
+    protected Dictionary<string, double> maxReward;
+    protected readonly bool useCanonicalState;
+    protected readonly IProblem problem;
+
+    protected GrammarPolicy(IProblem problem, bool useCanonicalState = false) {
+      this.useCanonicalState = useCanonicalState;
+      this.problem = problem;
+      this.tries = new Dictionary<string, int>();
+      this.avgReward = new Dictionary<string, double>();
+      this.maxReward = new Dictionary<string, double>();
+    }
+
+    public abstract bool TrySelect(Random random, string curState, IEnumerable<string> afterStates, out int selectedStateIdx);
+
+    public virtual void UpdateReward(IEnumerable<string> stateTrajectory, double reward) {
+      foreach (var state in stateTrajectory) {
+        var s = CanonicalState(state);
+
+        if (!tries.ContainsKey(s)) tries.Add(s, 0);
+        if (!avgReward.ContainsKey(s)) avgReward.Add(s, 0);
+        if (!maxReward.ContainsKey(s)) maxReward.Add(s, 0);
+
+        tries[s]++;
+        double alpha = 1.0 / tries[s];
+        avgReward[s] += alpha * (reward - avgReward[s]);
+        maxReward[s] = Math.Max(maxReward[s], reward);
+      }
+    }
+
+    public virtual void Reset() {
+      avgReward.Clear();
+      maxReward.Clear();
+      tries.Clear();
+    }
+
+    public double AvgReward(string state) {
+      var s = CanonicalState(state);
+      if (avgReward.ContainsKey(s)) return avgReward[s];
+      else return 0.0;
+    }
+
+    public double MaxReward(string state) {
+      var s = CanonicalState(state);
+      if (maxReward.ContainsKey(s)) return maxReward[s];
+      else return 0.0;
+    }
+
+    public virtual int GetTries(string state) {
+      var s = CanonicalState(state);
+      if (tries.ContainsKey(s)) return tries[s];
+      else return 0;
+    }
+
+    public virtual double GetValue(string state) {
+      return AvgReward(state);
+    }
+
+    protected string CanonicalState(string state) {
+      if (useCanonicalState) return problem.CanonicalRepresentation(state);
+      else return state;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/IGrammarPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/IGrammarPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/IGrammarPolicy.cs	(revision 12290)
@@ -0,0 +1,12 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.Bandits.GrammarPolicies {
+  public interface IGrammarPolicy : ISequentialDecisionPolicy<string> {
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/RandomPolicy.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/RandomPolicy.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SequentialDecisionPolicies/RandomPolicy.cs	(revision 12290)
@@ -0,0 +1,21 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.Bandits.GrammarPolicies {
+  public class RandomPolicy : GrammarPolicy {
+    public RandomPolicy(IProblem problem, bool useCanonicalRepresentation)
+      : base(problem, useCanonicalRepresentation) {
+    }
+
+    public override bool TrySelect(Random random, string curState, IEnumerable<string> afterStates, out int selectedStateIdx) {
+      // never fail => allows re-visits of terminal states
+      selectedStateIdx = random.Next(afterStates.Count());
+      return true;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SolverBase.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SolverBase.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/SolverBase.cs	(revision 12290)
@@ -0,0 +1,32 @@
+﻿using System;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GrammaticalOptimization {
+  public abstract class SolverBase : ISolver {
+    public event Action<string, double> FoundNewBestSolution;
+    public event Action<string, double> SolutionEvaluated;
+
+    protected double bestQuality = double.NegativeInfinity;
+
+    protected SolverBase() { }
+
+    public abstract void Run(int maxEvaluations);
+
+    protected virtual void OnSolutionEvaluated(string sentence, double quality) {
+      RaiseSolutionEvaluated(sentence, quality);
+      if (quality > bestQuality) {
+        bestQuality = quality;
+        RaiseFoundNewBestSolution(sentence, quality);
+      }
+    }
+
+    private void RaiseSolutionEvaluated(string sentence, double quality) {
+      var handler = SolutionEvaluated;
+      if (handler != null) handler(sentence, quality);
+    }
+    private void RaiseFoundNewBestSolution(string sentence, double quality) {
+      var handler = FoundNewBestSolution;
+      if (handler != null) handler(sentence, quality);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveBreadthFirstSearch.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveBreadthFirstSearch.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveBreadthFirstSearch.cs	(revision 12290)
@@ -0,0 +1,68 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GrammaticalOptimization {
+  public class ExhaustiveBreadthFirstSearch {
+    public event Action<string, double> FoundNewBestSolution;
+    public event Action<string, double> SolutionEvaluated;
+
+    private readonly int maxLen;
+    private readonly Queue<Sequence> bfsQueue = new Queue<Sequence>();
+    private readonly IProblem problem;
+
+    public ExhaustiveBreadthFirstSearch(IProblem problem, int maxLen) {
+      this.problem = problem;
+      this.maxLen = maxLen;
+    }
+
+    public void Run(int maxIterations) {
+      double bestQuality = double.MinValue;
+      bfsQueue.Enqueue(new Sequence(problem.Grammar.SentenceSymbol));
+      var sentences = GenerateLanguage(problem.Grammar);
+      var sentenceEnumerator = sentences.GetEnumerator();
+      for (int i = 0; sentenceEnumerator.MoveNext() && i < maxIterations; i++) {
+        var sentence = sentenceEnumerator.Current.ToString();
+        var quality = problem.Evaluate(sentence) / problem.BestKnownQuality(maxLen);
+        RaiseSolutionEvaluated(sentence, quality);
+
+        if (quality > bestQuality) {
+          bestQuality = quality;
+          RaiseFoundNewBestSolution(sentence, quality);
+        }
+      }
+    }
+
+    // create sentences lazily
+    private IEnumerable<Sequence> GenerateLanguage(IGrammar grammar) {
+      while (bfsQueue.Any()) {
+        var phrase = bfsQueue.Dequeue();
+
+        char nt = phrase.FirstNonTerminal;
+
+        var alts = grammar.GetAlternatives(nt);
+        foreach (var alt in alts) {
+          var newPhrase = new Sequence(phrase);
+          newPhrase.ReplaceAt(newPhrase.FirstNonTerminalIndex, 1, alt);
+          if (newPhrase.IsTerminal && newPhrase.Length <= maxLen) {
+            yield return newPhrase;
+          } else if (grammar.MinPhraseLength(newPhrase) <= maxLen) {
+            bfsQueue.Enqueue(newPhrase);
+          }
+        }
+      }
+    }
+
+
+    private void RaiseSolutionEvaluated(string sentence, double quality) {
+      var handler = SolutionEvaluated;
+      if (handler != null) handler(sentence, quality);
+    }
+    private void RaiseFoundNewBestSolution(string sentence, double quality) {
+      var handler = FoundNewBestSolution;
+      if (handler != null) handler(sentence, quality);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveDepthFirstSearch.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveDepthFirstSearch.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveDepthFirstSearch.cs	(revision 12290)
@@ -0,0 +1,68 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GrammaticalOptimization {
+  public class ExhaustiveDepthFirstSearch {
+    public event Action<string, double> FoundNewBestSolution;
+    public event Action<string, double> SolutionEvaluated;
+
+    private readonly int maxLen;
+    private readonly Stack<Sequence> stack = new Stack<Sequence>();
+    private readonly IProblem problem;
+
+    public ExhaustiveDepthFirstSearch(IProblem problem, int maxLen) {
+      this.maxLen = maxLen;
+      this.problem = problem;
+    }
+
+    public void Run(int maxIterations) {
+      double bestQuality = double.MinValue;
+      stack.Push(new Sequence(problem.Grammar.SentenceSymbol));
+      var sentences = GenerateLanguage(problem.Grammar);
+      var sentenceEnumerator = sentences.GetEnumerator();
+      for (int i = 0; sentenceEnumerator.MoveNext() && i < maxIterations; i++) {
+        var sentence = sentenceEnumerator.Current.ToString();
+        var quality = problem.Evaluate(sentence) / problem.BestKnownQuality(maxLen);
+        RaiseSolutionEvaluated(sentence, quality);
+
+        if (quality > bestQuality) {
+          bestQuality = quality;
+          RaiseFoundNewBestSolution(sentence, quality);
+        }
+      }
+    }
+
+    // create sentences lazily
+    private IEnumerable<Sequence> GenerateLanguage(IGrammar grammar) {
+      while (stack.Any()) {
+        var phrase = stack.Pop();
+
+        char nt = phrase.FirstNonTerminal;
+        var alts = grammar.GetAlternatives(nt);
+        foreach (var alt in alts) {
+          var newPhrase = new Sequence(phrase);
+          newPhrase.ReplaceAt(newPhrase.FirstNonTerminalIndex, 1, alt);
+
+          if (newPhrase.IsTerminal && newPhrase.Length <= maxLen) {
+            yield return newPhrase;
+          } else if (grammar.MinPhraseLength(newPhrase) <= maxLen) {
+            stack.Push(newPhrase);
+          }
+        }
+      }
+    }
+
+
+    private void RaiseSolutionEvaluated(string sentence, double quality) {
+      var handler = SolutionEvaluated;
+      if (handler != null) handler(sentence, quality);
+    }
+    private void RaiseFoundNewBestSolution(string sentence, double quality) {
+      var handler = FoundNewBestSolution;
+      if (handler != null) handler(sentence, quality);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveRandomFirstSearch.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveRandomFirstSearch.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/ExhaustiveRandomFirstSearch.cs	(revision 12290)
@@ -0,0 +1,74 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GrammaticalOptimization {
+  public class ExhaustiveRandomFirstSearch {
+    public event Action<string, double> FoundNewBestSolution;
+    public event Action<string, double> SolutionEvaluated;
+
+    private readonly int maxLen;
+    private readonly System.Collections.Generic.SortedList<double, Sequence> sortedList = new SortedList<double, Sequence>();
+    private readonly IProblem problem;
+    private readonly Random random;
+
+    public ExhaustiveRandomFirstSearch(IProblem problem, Random random, int maxLen) {
+      this.maxLen = maxLen;
+      this.problem = problem;
+      this.random = new Random();
+    }
+
+    public void Run(int maxIterations) {
+      double bestQuality = double.MinValue;
+      sortedList.Add(0, new Sequence(problem.Grammar.SentenceSymbol));
+      var sentences = GenerateLanguage(problem.Grammar);
+      var sentenceEnumerator = sentences.GetEnumerator();
+      for (int i = 0; sentenceEnumerator.MoveNext() && i < maxIterations; i++) {
+        var sentence = sentenceEnumerator.Current.ToString();
+        var quality = problem.Evaluate(sentence) / problem.BestKnownQuality(maxLen);
+        RaiseSolutionEvaluated(sentence, quality);
+
+        if (quality > bestQuality) {
+          bestQuality = quality;
+          RaiseFoundNewBestSolution(sentence, quality);
+        }
+      }
+    }
+
+    // create sentences lazily
+    private IEnumerable<Sequence> GenerateLanguage(IGrammar grammar) {
+      while (sortedList.Any()) {
+        var phrase = sortedList.First().Value;
+        sortedList.RemoveAt(0);
+
+        char nt = phrase.FirstNonTerminal;
+        var alts = grammar.GetAlternatives(nt);
+        foreach (var alt in alts) {
+          var newPhrase = new Sequence(phrase);
+          newPhrase.ReplaceAt(newPhrase.FirstNonTerminalIndex, 1, alt);
+
+          if (newPhrase.IsTerminal && newPhrase.Length <= maxLen) {
+            yield return newPhrase;
+          } else if (grammar.MinPhraseLength(newPhrase) <= maxLen) {
+            var r = random.NextDouble();
+            Sequence t;
+            while (sortedList.TryGetValue(r, out t)) { r = random.NextDouble(); }
+            sortedList.Add(r, newPhrase);
+          }
+        }
+      }
+    }
+
+
+    private void RaiseSolutionEvaluated(string sentence, double quality) {
+      var handler = SolutionEvaluated;
+      if (handler != null) handler(sentence, quality);
+    }
+    private void RaiseFoundNewBestSolution(string sentence, double quality) {
+      var handler = FoundNewBestSolution;
+      if (handler != null) handler(sentence, quality);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/RandomSearch.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/RandomSearch.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/RandomSearch.cs	(revision 12290)
@@ -0,0 +1,29 @@
+﻿using System;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GrammaticalOptimization {
+  public class RandomSearch : SolverBase {
+    private readonly int maxLen;
+    private readonly Random random;
+    private readonly IProblem problem;
+
+    public RandomSearch(IProblem problem, Random random, int maxLen) {
+      this.maxLen = maxLen;
+      this.random = random;
+      this.problem = problem;
+    }
+
+    public override void Run(int maxIterations) {
+      for (int i = 0; i < maxIterations; i++) {
+        var sentence = CreateSentence(problem.Grammar).ToString();
+        var quality = problem.Evaluate(sentence) / problem.BestKnownQuality(maxLen);
+        OnSolutionEvaluated(sentence, quality);
+      }
+    }
+
+    private Sequence CreateSentence(IGrammar grammar) {
+      var sentence = new Sequence(grammar.SentenceSymbol);
+      return grammar.CompleteSentenceRandomly(random, sentence, maxLen);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/SequentialSearch.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/SequentialSearch.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.GrammaticalOptimization/Solvers/SequentialSearch.cs	(revision 12290)
@@ -0,0 +1,245 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Resources;
+using System.Runtime.InteropServices;
+using System.Text;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.GrammaticalOptimization {
+  // a search procedure that uses a policy to generate sentences and updates the policy (online RL)
+  // 1) Start with phrase = sentence symbol of grammar
+  // 2) Repeat
+  //    a) generate derived phrases using left-canonical derivation and grammar rules
+  //    b) keep only the phrases which are allowed (sentence length limit)
+  //    c) if the set of phrases is empty restart with 1)
+  //    d) otherwise use policy to select one of the possible derived phrases as active phrase
+  //       the policy has the option to fail (for instance if all derived phrases are terminal and should not be visited again), in this case we restart at 1
+  //    ... until phrase is terminal
+  // 3) Collect reward and update policy (feedback: state of visited rewards from step 2)
+  public class SequentialSearch : SolverBase {
+    // only for storing states so that it is not necessary to allocate new state strings whenever we select a follow state using the policy
+    private class TreeNode {
+      public int randomTries;
+      public string phrase;
+      public Sequence alternative;
+      public TreeNode[] children;
+
+      public TreeNode(string phrase, Sequence alternative) {
+        this.alternative = alternative;
+        this.phrase = phrase;
+      }
+    }
+
+
+    private readonly int maxLen;
+    private readonly IProblem problem;
+    private readonly Random random;
+    private readonly int randomTries;
+    private readonly IGrammarPolicy behaviourPolicy;
+    private TreeNode rootNode;
+
+    private int tries;
+    private int maxSearchDepth;
+
+    private string bestPhrase;
+    private readonly List<string> stateChain;
+
+    public SequentialSearch(IProblem problem, int maxLen, Random random, int randomTries, IGrammarPolicy behaviourPolicy) {
+      this.maxLen = maxLen;
+      this.problem = problem;
+      this.random = random;
+      this.randomTries = randomTries;
+      this.behaviourPolicy = behaviourPolicy;
+      this.stateChain = new List<string>();
+    }
+
+    public bool StopRequested {
+      get;
+      set;
+    }
+
+    public override void Run(int maxIterations) {
+      Reset();
+
+      for (int i = 0; !StopRequested && !Done() && i < maxIterations; i++) {
+        var phrase = SampleSentence(problem.Grammar);
+        // can fail on the last sentence
+        if (phrase.IsTerminal) {
+          var sentence = phrase.ToString();
+          tries++;
+          var quality = problem.Evaluate(sentence) / problem.BestKnownQuality(maxLen);
+          if (double.IsNaN(quality)) quality = 0.0;
+          Debug.Assert(quality >= 0 && quality <= 1.0);
+
+          if (quality > bestQuality) {
+            bestPhrase = sentence;
+          }
+
+          OnSolutionEvaluated(sentence, quality);
+          DistributeReward(quality);
+
+        }
+      }
+    }
+
+
+    private Sequence SampleSentence(IGrammar grammar) {
+      Sequence phrase;
+      do {
+        stateChain.Clear();
+        phrase = new Sequence(rootNode.phrase);
+      } while (!Done() && !TryCompleteSentence(grammar, ref phrase));
+      return phrase;
+    }
+
+    private bool TryCompleteSentence(IGrammar g, ref Sequence phrase) {
+      if (phrase.Length > maxLen) throw new ArgumentException();
+      if (g.MinPhraseLength(phrase) > maxLen) throw new ArgumentException();
+      var curDepth = 0;
+      var n = rootNode;
+      stateChain.Add(n.phrase);
+
+      while (!phrase.IsTerminal) {
+        if (n.randomTries < randomTries) {
+          n.randomTries++;
+          maxSearchDepth = Math.Max(maxSearchDepth, curDepth);
+          g.CompleteSentenceRandomly(random, phrase, maxLen);
+          return true;
+        } else {
+          // => select using bandit policy
+          // failure means we simply restart
+          GenerateFollowStates(n); // creates child nodes for node n
+
+          int selectedChildIdx;
+          if (!behaviourPolicy.TrySelect(random, n.phrase, n.children.Select(ch => ch.phrase), out selectedChildIdx)) {
+            return false;
+          }
+          phrase.ReplaceAt(phrase.FirstNonTerminalIndex, 1, n.children[selectedChildIdx].alternative);
+
+          // prepare for next iteration
+          n = n.children[selectedChildIdx];
+          stateChain.Add(n.phrase);
+          curDepth++;
+        }
+      } // while
+
+      maxSearchDepth = Math.Max(maxSearchDepth, curDepth);
+      return true;
+    }
+
+
+    private IEnumerable<string> GenerateFollowStates(TreeNode n) {
+      // create children on the first visit
+      if (n.children == null) {
+        var g = problem.Grammar;
+        // tree is only used for easily retrieving the follow-states of a state
+        var phrase = new Sequence(n.phrase);
+        char nt = phrase.FirstNonTerminal;
+
+        int maxLenOfReplacement = maxLen - (phrase.Length - 1);
+        // replacing aAb with maxLen 4 means we can only use alternatives with a minPhraseLen <= 2
+        Debug.Assert(maxLenOfReplacement > 0);
+
+        var alts = g.GetAlternatives(nt).Where(alt => g.MinPhraseLength(alt) <= maxLenOfReplacement);
+
+        var children = new TreeNode[alts.Count()];
+        int idx = 0;
+        foreach (var alt in alts) {
+          // var newPhrase = new Sequence(phrase); // clone
+          // newPhrase.ReplaceAt(newPhrase.FirstNonTerminalIndex, 1, alt);
+          // children[idx++] = new TreeNode(newPhrase.ToString(), alt);
+
+          // since we are not using a sequence later on we might directly transform the current sequence to a string and replace there
+          var phraseStr = phrase.ToString();
+          var sb = new StringBuilder(phraseStr);
+          sb.Remove(phrase.FirstNonTerminalIndex, 1).Insert(phrase.FirstNonTerminalIndex, alt.ToString());
+          children[idx++] = new TreeNode(sb.ToString(), alt);
+        }
+        n.children = children;
+      }
+      return n.children.Select(ch => ch.phrase);
+    }
+
+    private void DistributeReward(double reward) {
+      behaviourPolicy.UpdateReward(stateChain, reward);
+    }
+
+
+    private void Reset() {
+      StopRequested = false;
+      behaviourPolicy.Reset();
+      maxSearchDepth = 0;
+      bestQuality = 0.0;
+      tries = 0;
+      rootNode = new TreeNode(problem.Grammar.SentenceSymbol.ToString(), new ReadonlySequence("$"));
+    }
+
+    public bool Done() {
+      int selectedStateIdx;
+      return !behaviourPolicy.TrySelect(random, rootNode.phrase, GenerateFollowStates(rootNode), out selectedStateIdx);
+    }
+
+    #region introspection
+    public void PrintStats() {
+      Console.WriteLine("depth: {0,5} tries: {1,5} best phrase {2,50} bestQ {3:F3}", maxSearchDepth, tries, bestPhrase, bestQuality);
+
+      // use behaviour strategy to generate the currently prefered sentence
+      var policy = behaviourPolicy;
+
+      var n = rootNode;
+
+      while (n != null) {
+        var phrase = n.phrase;
+        Console.ForegroundColor = ConsoleColor.White;
+        Console.WriteLine("{0,-30}", phrase);
+        var children = n.children;
+        if (children == null || !children.Any()) break;
+        var values = children.Select(ch => policy.GetValue(ch.phrase));
+        var maxValue = values.Max();
+        if (maxValue == 0) maxValue = 1.0;
+        if (double.IsPositiveInfinity(maxValue)) maxValue = double.MaxValue;
+
+        // write phrases
+        foreach (var ch in children) {
+          SetColorForValue(policy.GetValue(ch.phrase) / maxValue);
+          Console.Write(" {0,-4}", ch.phrase.Substring(Math.Max(0, ch.phrase.Length - 3), Math.Min(3, ch.phrase.Length)));
+        }
+        Console.WriteLine();
+
+        // write values
+        foreach (var ch in children) {
+          SetColorForValue(policy.GetValue(ch.phrase) / maxValue);
+          Console.Write(" {0:F2}", policy.GetValue(ch.phrase) * 10.0);
+        }
+        Console.WriteLine();
+
+        // write tries
+        foreach (var ch in children) {
+          SetColorForValue(policy.GetValue(ch.phrase) / maxValue);
+          Console.Write(" {0,4}", policy.GetTries(ch.phrase));
+        }
+        Console.WriteLine();
+        int selectedChildIdx;
+        if (!policy.TrySelect(random, phrase, children.Select(ch => ch.phrase), out selectedChildIdx)) {
+          break;
+        }
+        n = n.children[selectedChildIdx];
+      }
+
+      Console.ForegroundColor = ConsoleColor.White;
+      Console.WriteLine("-------------------");
+    }
+
+    private void SetColorForValue(double v) {
+      Console.ForegroundColor = ConsoleEx.ColorForValue(v);
+    }
+    #endregion
+
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Base/TreeNode.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Base/TreeNode.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Base/TreeNode.cs	(revision 12290)
@@ -0,0 +1,35 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.MonteCarloTreeSearch.Base
+{
+    public class TreeNode
+    {
+        public string phrase;
+        public TreeNode parent;
+        public List<TreeNode> children;
+        public IBanditPolicyActionInfo actionInfo;
+
+        public TreeNode(TreeNode parent, string phrase)
+        {
+            this.parent = parent;
+            this.phrase = phrase;
+            actionInfo = new DefaultPolicyActionInfo();
+        }
+        public bool IsLeaf()
+        {
+            return children == null || !children.Any();
+        }
+
+        internal IEnumerable<IBanditPolicyActionInfo> GetChildActionInfos()
+        {
+            return children.Select(n => n.actionInfo);
+        }
+    }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/HeuristicLab.Algorithms.MonteCarloTreeSearch.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/HeuristicLab.Algorithms.MonteCarloTreeSearch.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/HeuristicLab.Algorithms.MonteCarloTreeSearch.csproj	(revision 12290)
@@ -0,0 +1,74 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{2C115235-8FA9-4F7F-B3A0-A0144F8A35CA}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Algorithms.MonteCarloTreeSearch</RootNamespace>
+    <AssemblyName>HeuristicLab.Algorithms.MonteCarloTreeSearch</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Simulation\ISimulation.cs" />
+    <Compile Include="MonteCarloTreeSearch.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Base\TreeNode.cs" />
+    <Compile Include="Simulation\RandomSimulation.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.Bandits\HeuristicLab.Algorithms.Bandits.csproj">
+      <Project>{24408f7d-ee0f-4886-a08b-ec324d662e47}</Project>
+      <Name>HeuristicLab.Algorithms.Bandits</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.GrammaticalOptimization\HeuristicLab.Algorithms.GrammaticalOptimization.csproj">
+      <Project>{eea07488-1a51-412a-a52c-53b754a628b3}</Project>
+      <Name>HeuristicLab.Algorithms.GrammaticalOptimization</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3a2fbbcb-f9df-4970-87f3-f13337d941ad}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.csproj">
+      <Project>{cb9dccf6-667e-4a13-b82d-dbd6b45a045e}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/MonteCarloTreeSearch.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/MonteCarloTreeSearch.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/MonteCarloTreeSearch.cs	(revision 12290)
@@ -0,0 +1,176 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Algorithms.MonteCarloTreeSearch.Base;
+using HeuristicLab.Algorithms.MonteCarloTreeSearch.Simulation;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.MonteCarloTreeSearch
+{
+    public class MonteCarloTreeSearch : SolverBase
+    {
+        private readonly int maxLen;
+        private readonly IProblem problem;
+        private readonly IGrammar grammar;
+        private readonly Random random;
+        private readonly IBanditPolicy behaviourPolicy;
+        private readonly ISimulation simulation;
+        private TreeNode rootNode;
+
+        public MonteCarloTreeSearch(IProblem problem, int maxLen, Random random, IBanditPolicy behaviourPolicy, ISimulation simulationPolicy)
+        {
+            this.problem = problem;
+            this.grammar = problem.Grammar;
+            this.maxLen = maxLen;
+            this.random = random;
+            this.behaviourPolicy = behaviourPolicy;
+            this.simulation = simulationPolicy;
+        }
+
+        public bool StopRequested
+        {
+            get;
+            set;
+        }
+
+        public override void Run(int maxIterations)
+        {
+            Reset();
+            for (int i = 0; !StopRequested && i < maxIterations; i++)
+            {
+                TreeNode currentNode = rootNode;
+
+                while (!currentNode.IsLeaf())
+                {
+                    int currentActionIndex = behaviourPolicy.SelectAction(random,
+                        currentNode.GetChildActionInfos());
+                    currentNode = currentNode.children[currentActionIndex];
+                }
+
+                string phrase = currentNode.phrase;
+
+                if (!grammar.IsTerminal(phrase))
+                {
+                    ExpandTreeNode(currentNode);
+
+                    currentNode =
+                        currentNode.children[behaviourPolicy.SelectAction(random, currentNode.GetChildActionInfos())];
+                }
+                double quality = simulation.Simulate(currentNode);
+                OnSolutionEvaluated(phrase, quality);
+
+                Propagate(currentNode, quality);
+            }
+        }
+
+        private void ExpandTreeNode(TreeNode treeNode)
+        {
+            // create children on the first visit
+            if (treeNode.children == null)
+            {
+                treeNode.children = new List<TreeNode>();
+
+                var phrase = new Sequence(treeNode.phrase);
+                // create subnodes for each nt-symbol in phrase
+                for (int i = 0; i < phrase.Length; i++)
+                {
+                    char symbol = phrase[i];
+                    if (grammar.IsNonTerminal(symbol))
+                    {
+                        // create subnode for each alternative of symbol
+                        foreach (Sequence alternative in grammar.GetAlternatives(symbol))
+                        {
+                            Sequence newSequence = new Sequence(phrase);
+                            newSequence.ReplaceAt(i, 1, alternative);
+                            if (newSequence.Length <= maxLen)
+                            {
+                                TreeNode childNode = new TreeNode(treeNode, newSequence.ToString());
+                                treeNode.children.Add(childNode);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        private void Reset()
+        {
+            StopRequested = false;
+            bestQuality = 0.0;
+            rootNode = new TreeNode(null, grammar.SentenceSymbol.ToString());
+        }
+
+        private void Propagate(TreeNode node, double quality)
+        {
+            var currentNode = node;
+            do
+            {
+                currentNode.actionInfo.UpdateReward(quality);
+                currentNode = currentNode.parent;
+            } while (currentNode != null);
+        }
+
+        public void PrintStats()
+        {
+            //Console.WriteLine("depth: {0,5} tries: {1,5} best phrase {2,50} bestQ {3:F3}", maxSearchDepth, tries, bestPhrase, bestQuality);
+
+            //// use behaviour strategy to generate the currently prefered sentence
+            //var policy = behaviourPolicy;
+
+            //var n = rootNode;
+
+            //while (n != null)
+            //{
+            //    var phrase = n.phrase;
+            //    Console.ForegroundColor = ConsoleColor.White;
+            //    Console.WriteLine("{0,-30}", phrase);
+            //    var children = n.children;
+            //    if (children == null || !children.Any()) break;
+            //    var values = children.Select(ch => policy.GetValue(ch.phrase));
+            //    var maxValue = values.Max();
+            //    if (maxValue == 0) maxValue = 1.0;
+
+            //    // write phrases
+            //    foreach (var ch in children)
+            //    {
+            //        SetColorForValue(policy.GetValue(ch.phrase) / maxValue);
+            //        Console.Write(" {0,-4}", ch.phrase.Substring(Math.Max(0, ch.phrase.Length - 3), Math.Min(3, ch.phrase.Length)));
+            //    }
+            //    Console.WriteLine();
+
+            //    // write values
+            //    foreach (var ch in children)
+            //    {
+            //        SetColorForValue(policy.GetValue(ch.phrase) / maxValue);
+            //        Console.Write(" {0:F2}", policy.GetValue(ch.phrase) * 10.0);
+            //    }
+            //    Console.WriteLine();
+
+            //    // write tries
+            //    foreach (var ch in children)
+            //    {
+            //        SetColorForValue(policy.GetValue(ch.phrase) / maxValue);
+            //        Console.Write(" {0,4}", policy.GetTries(ch.phrase));
+            //    }
+            //    Console.WriteLine();
+            //    int selectedChildIdx;
+            //    if (!policy.TrySelect(random, phrase, children.Select(ch => ch.phrase), out selectedChildIdx))
+            //    {
+            //        break;
+            //    }
+            //    n = n.children[selectedChildIdx];
+            //}
+
+            //Console.ForegroundColor = ConsoleColor.White;
+            //Console.WriteLine("-------------------");
+        }
+
+        private void SetColorForValue(double v)
+        {
+            Console.ForegroundColor = ConsoleEx.ColorForValue(v);
+        }
+    }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Algorithms.MonteCarloTreeSearch")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("HeuristicLab.Algorithms.MonteCarloTreeSearch")]
+[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("3482040b-e928-4464-8b3c-460a5886a18d")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Simulation/ISimulation.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Simulation/ISimulation.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Simulation/ISimulation.cs	(revision 12290)
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Algorithms.MonteCarloTreeSearch.Base;
+
+namespace HeuristicLab.Algorithms.MonteCarloTreeSearch.Simulation
+{
+    public interface ISimulation
+    {
+        double Simulate(TreeNode node);
+    }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Simulation/RandomSimulation.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Simulation/RandomSimulation.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Algorithms.MonteCarloTreeSearch/Simulation/RandomSimulation.cs	(revision 12290)
@@ -0,0 +1,53 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using HeuristicLab.Algorithms.MonteCarloTreeSearch.Base;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+namespace HeuristicLab.Algorithms.MonteCarloTreeSearch.Simulation
+{
+    public class RandomSimulation : ISimulation
+    {
+        private IProblem problem;
+        private IGrammar grammar;
+        private Random random;
+        private int maxLen;
+
+        public RandomSimulation(IProblem problem, Random random, int maxLen)
+        {
+            this.problem = problem;
+            this.grammar = problem.Grammar;
+            this.random = random;
+            this.maxLen = maxLen;
+        }
+
+        public double Simulate(TreeNode node)
+        {
+            Sequence sequence = new Sequence(node.phrase);
+            List<int> ntPositions = new List<int>();
+            while (!sequence.IsTerminal)
+            {
+                // select one random nt-symbol
+                ntPositions.Clear();
+                for (int i = 0; i < sequence.Length; i++)
+                {
+                    if (grammar.IsNonTerminal(sequence[i]))
+                    {
+                        ntPositions.Add(i);
+                    }
+                }
+                int ntSymbolIndex = ntPositions[random.Next(ntPositions.Count)];
+                char ntSymbol = sequence[ntSymbolIndex];
+
+                // select one random alternative for nt-symbol
+                IEnumerable<Sequence> alternatives = grammar.GetAlternatives(ntSymbol);
+                Sequence alternative = alternatives.ElementAt(random.Next(alternatives.Count()));
+                if (sequence.Length + alternative.Length - 1 <= maxLen)
+                {
+                    sequence.ReplaceAt(ntSymbolIndex, 1, alternative);
+                }
+            }
+            return problem.Evaluate(sequence.ToString());
+        }
+    }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/ConsoleEx.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/ConsoleEx.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/ConsoleEx.cs	(revision 12290)
@@ -0,0 +1,189 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Drawing;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Common {
+  // for controlling console colors
+  // see output of Solvers (PrintStats)
+  // http://stackoverflow.com/questions/25274019/strange-setconsolescreenbufferinfoex-behavior
+  public class ConsoleEx {
+
+    private static readonly ConsoleColor[] colors = (ConsoleColor[])Enum.GetValues(typeof(ConsoleColor));
+
+    static ConsoleEx() {
+      // init colors
+      /*
+IntPtr handle = CreateFile("CONOUT$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, null, 3, 0, IntPtr.Zero);
+
+CONSOLE_SCREEN_BUFFER_INFO_EX currentScreen = new CONSOLE_SCREEN_BUFFER_INFO_EX();
+currentScreen.cbSize = (uint)Marshal.SizeOf(currentScreen);
+
+GetConsoleScreenBufferInfoEx(handle, ref currentScreen);
+
+currentScreen.ColorF = currentScreen.Color0;
+currentScreen.dwSizeX = 80;
+currentScreen.dwSizeY = 30;
+
+SetConsoleScreenBufferInfoEx(handle, ref currentScreen);
+CloseHandle(handle);*/
+      // Initializing Screen with Testcase
+
+      IntPtr handle = CreateFile("CONOUT$", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, null, 3, 0, IntPtr.Zero);
+      CONSOLE_SCREEN_BUFFER_INFO_EX currentScreen = new CONSOLE_SCREEN_BUFFER_INFO_EX();
+      currentScreen.cbSize = (uint)Marshal.SizeOf(currentScreen);
+      GetConsoleScreenBufferInfoEx(handle, ref currentScreen);
+
+
+      CONSOLE_SCREEN_BUFFER_INFO_EX newData = currentScreen;
+
+      // newData.srWindowRight = 300;
+      // newData.srWindowBottom = 300;
+
+      double startRed = 0;
+      double endRed = 1.41;
+
+      double startGreen = -1.41;
+      double endGreen = 1.41;
+
+      double startBlue = -3;
+      double endBlue = 1;
+
+      double startAlpha = 0;
+      double endAlpha = 0;
+
+      // fire
+      //var red = new int[] { 1, 49, 98, 146, 173, 195, 217, 240, 255, 255, 255, 255, 255, 255, 255 };
+      //var green = new int[] { 0, 0, 0, 0, 0, 0, 35, 79, 117, 147, 175, 205, 234, 255, 255 };
+      //var blue = new int[] { 96, 165, 220, 210, 151, 93, 35, 0, 0, 0, 0, 0, 0, 98, 223 };
+      var red = new int[16];
+      var green = new int[16];
+      var blue = new int[16];
+      var alpha = new int[16];
+
+      var r = startRed;
+      var g = startGreen;
+      var b = startBlue;
+      var a = startAlpha;
+      var n = 15;
+      for (int i = 0; i < n; i++) {
+        red[i] = (int)Math.Round(Math.Max(Math.Min(1.0, r), 0) * 255);
+        green[i] = (int)Math.Round(Math.Max(Math.Min(1.0, g), 0) * 255);
+        blue[i] = (int)Math.Round(Math.Max(Math.Min(1.0, b), 0) * 255);
+        alpha[i] = (int)Math.Round(Math.Max(Math.Min(1.0, a), 0) * 255);
+
+        r += (endRed - startRed) / n;
+        g += (endGreen - startGreen) / n;
+        b += (endBlue - startBlue) / n;
+        a += (endAlpha - startAlpha) / n;
+      }
+
+      newData.ColorF = newData.Color7;
+      int h = 0;
+      newData.Color0 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color1 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color2 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color3 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color4 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color5 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color6 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color7 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color8 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.Color9 = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.ColorA = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.ColorB = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.ColorC = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.ColorD = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h]; h++;
+      newData.ColorE = (alpha[h] << 24) + (blue[h] << 16) + (green[h] << 8) + red[h];
+      // newData.ColorF = (r << 16) + (g << 8) + b;
+
+      SetConsoleScreenBufferInfoEx(handle, ref newData);
+      Console.ForegroundColor = ConsoleColor.White;
+
+      CloseHandle(handle);
+    }
+
+    public static ConsoleColor ColorForValue(double d) {
+      //Debug.Assert(d >= 0 && d <= 1.0);
+      d = Math.Min(1.0, Math.Max(0.0, d));
+      var cIdx = Math.Max(0, (int)Math.Floor(d * 15) - 1);
+      return colors[cIdx];
+    }
+
+
+
+
+    [DllImport("kernel32.dll", SetLastError = true)]
+    public static extern bool GetConsoleScreenBufferInfoEx(IntPtr handle, ref CONSOLE_SCREEN_BUFFER_INFO_EX consoleScreenBufferInfoEx);
+    [DllImport("kernel32.dll", SetLastError = true)]
+    static extern bool SetConsoleScreenBufferInfoEx(IntPtr handle, ref CONSOLE_SCREEN_BUFFER_INFO_EX consoleScreenBufferInfoEx);
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct CONSOLE_SCREEN_BUFFER_INFO_EX {
+      public uint cbSize;  // 96
+      public short dwSizeX;
+      public short dwSizeY;
+      public short dwCursorPositionX;
+      public short dwCursorPositionY;
+      public short wAttributes;
+      public short srWindowLeft;
+      public short srWindowTop;
+      public short srWindowRight;
+      public short srWindowBottom;
+      public short dwMaximumWindowSizeX;
+      public short dwMaximumWindowSizeY;
+
+      public ushort wPopupAttributes;
+      public bool bFullscreenSupported;
+
+      public int Color0;
+      public int Color1;
+      public int Color2;
+      public int Color3;
+
+      public int Color4;
+      public int Color5;
+      public int Color6;
+      public int Color7;
+
+      public int Color8;
+      public int Color9;
+      public int ColorA;
+      public int ColorB;
+
+      public int ColorC;
+      public int ColorD;
+      public int ColorE;
+      public int ColorF;
+    }
+
+    [DllImport("kernel32.dll", SetLastError = true)]
+    private static extern IntPtr CreateFile(
+        string Filename,
+        int DesiredAccess,
+        int ShareMode,
+        [In][MarshalAs(UnmanagedType.LPStruct)]SecurityAttributes SecAttr,
+        int CreationDisposition,
+        int FlagsAndAttributes,
+        IntPtr TemplateFile);
+
+    private const int GENERIC_READ = unchecked((int)0x80000000);
+    private const int GENERIC_WRITE = 0x40000000;
+    private const int FILE_SHARE_READ = 1;
+    private const int FILE_SHARE_WRITE = 2;
+
+    [DllImport("kernel32.dll")]
+    private static extern bool CloseHandle(IntPtr handle);
+
+    [StructLayout(LayoutKind.Sequential)]
+    private class SecurityAttributes {
+      public int nLength;
+      public IntPtr lpSecurityDescriptor;
+      public bool bInheritHandle;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/ExpressionExtender.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/ExpressionExtender.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/ExpressionExtender.cs	(revision 12290)
@@ -0,0 +1,397 @@
+﻿using System;
+using System.CodeDom;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  // helper to create canonical forms of expressions
+  // TODO: change symbolicregressionpoly10problem to use this class
+  // this does not support expressions with constants (in transformations we assume constant opt is used)
+  // (e.g. we transform all negative signs to + because it is assumed a negative constant can be produced for the term)
+  public class ExpressionExtender {
+
+    // supports the grammar
+    // G(E):
+    //     E -> V | V+E | V-E | V*E | V%E | (E) 
+    //     V -> <variables>
+    //     ";
+
+    // might produce expressions of the form /x (= 1/x)
+    // the pipe symbol | is used for the constant one (in comparison to other constants)
+    private string sentence;
+    private int syIdx;
+
+    public string CanonicalRepresentation(string phrase) {
+      InitLex(phrase);
+      var e = CanonicalExpr();
+      return e.ToString();
+    }
+
+
+    private void InitLex(string sentence) {
+      this.sentence = sentence;
+      this.syIdx = 0;
+    }
+
+    private char CurSy() {
+      if (syIdx >= sentence.Length) return '\0';
+      return sentence[syIdx];
+    }
+    private void NewSy() {
+      if (syIdx < sentence.Length) syIdx++;
+    }
+
+    // an expression is a sorted set of terms 
+    // a term is an (ordered) list of factors
+    // a factor is either a single symbol or an inverted expression
+    // CanonicalExpression reads multiple terms (expressions) and must merge the terms in the expression (ordering and duplicates)
+    // CanonicalTerm reads multiple factors and must expand factors whenever it reads a combined factor (expression) it produces an expression
+    // CanonicalFactor produces an expression 
+
+
+    // canonical expression returns either a single term or a set of terms
+    private Expr CanonicalExpr() {
+      var terms = new List<Expr>();
+      terms.Add(CanonicalTerm());
+      var curSy = CurSy();
+      while (curSy == '+' || curSy == '-' || curSy == '^') {
+        if (curSy == '+') {
+          NewSy();
+          terms.Add(CanonicalTerm());
+        } else if (curSy == '-') {
+          NewSy();
+          terms.Add(CanonicalTerm()); // minus is the same as plus assuming constant opt
+        } else {
+          NewSy();
+          throw new NotImplementedException();
+          // var e = Expr(variables, constants);
+          // r = Not(r) * e + r * Not(e); // xor = (!x AND y) OR (x AND !y)
+        }
+        curSy = CurSy();
+      }
+
+      return new Expr(terms.SelectMany(t => t.Terms));
+    }
+
+    // canonical term returns either a single term (product of factors) or a set of terms 
+    private Expr CanonicalTerm() {
+      var factors = new List<Factor>();
+      var f = CanonicalFact();
+      if (f != null) factors.Add(f);
+      var curSy = CurSy();
+      while (curSy == '*' || curSy == '%') {
+        if (curSy == '*') {
+          NewSy();
+          f = CanonicalFact();
+          if (f != null) factors.Add(f);
+        } else {
+          NewSy();
+          f = CanonicalFact();
+          // if there is only one term we can add multiple inverted simple factors instead of the whole inverted expression
+          if (!f.IsSimpleFactor && f.Expr.Terms.Count == 1) {
+            foreach (var invF in f.Expr.Terms.First().Factors) {
+              if (invF.ToString() == "1") continue;
+              invF.Invert();
+              factors.Add(invF);
+            }
+          } else {
+            f.Invert();
+            if (f != null) factors.Add(f);
+          }
+        }
+        curSy = CurSy();
+      }
+
+      factors = CancelFactors(factors).ToList();
+      return ExpandFactors(factors);
+    }
+
+    // canonical fact returns a factor (either a singe variable, or a set of terms)
+    private Factor CanonicalFact() {
+      var curSy = CurSy();
+      if (curSy == '!') {
+        throw new NotSupportedException();
+      } else if (curSy == '(') {
+        NewSy();
+        Expr r = CanonicalExpr(); // this is already simplified
+        if (CurSy() != ')') throw new ArgumentException();
+        NewSy();
+        return new Factor(r);
+      } else if (curSy >= 'a' && curSy <= 'z') {
+        NewSy();
+        return new Factor(curSy);
+        //       } else if (curSy >= '0' && curSy <= '9') {
+      } else if (curSy >= 'A' && curSy <= 'Z') {
+        // treat nonterminals in the same way as variables
+        NewSy();
+        return new Factor(curSy);
+
+      } else throw new ArgumentException("found symbol " + curSy);
+    }
+
+    // a list of factors (symbols, or expressions, and possibly inverses are read
+    // a list to factors symbols or expressions and possibly inverses are produced
+    // all non-inverse expression factors are expanded
+    private Expr ExpandFactors(IEnumerable<Factor> factors) {
+      // if (invFactors.Count > 0) throw new NotImplementedException();
+      //Debug.Assert(!factors.First().IsInverse); // the first factor is never an inverted factor
+
+      // each factor could be a list of terms (expression)
+
+      Expr currentFact = null;
+      var firstFactor = factors.First();
+      if (firstFactor.IsSimpleFactor || firstFactor.IsInverse) currentFact = new Expr(new Term(firstFactor));
+      else currentFact = firstFactor.Expr;
+
+      foreach (var fact in factors.Skip(1)) {
+        Expr curExpr = null;
+        if (fact.IsSimpleFactor || fact.IsInverse) curExpr = new Expr(new Term(fact));
+        else curExpr = fact.Expr;
+        currentFact = AllProducts(currentFact, curExpr);
+      }
+      return currentFact;
+    }
+
+    private Expr AllProducts(Expr a, Expr b) {
+      var aTerms = a.Terms.ToArray();
+      var bTerms = b.Terms.ToArray();
+      var combs = from aT in aTerms
+                  from bT in bTerms
+                  let factors = CancelFactors(aT.Factors.Concat(bT.Factors))
+                  select new Term(factors);
+      return new Expr(combs);
+    }
+
+    private IEnumerable<Factor> CancelFactors(IEnumerable<Factor> factors) {
+      var factorsArr = factors.ToArray();
+      var results = new List<Factor>(factors);
+      foreach (var f in factorsArr) {
+        if (f.ToString() == "1") results.Remove(f);
+        if (f.ToString() == "1/(1)") results.Remove(f);
+        if (f.IsInverse) {
+          // find matching
+          Factor match;
+          match = factorsArr.FirstOrDefault(other => !other.IsInverse && f.Cancels(other));
+          if (match != null) {
+            results.Remove(f);
+            var idx = results.IndexOf(match);
+
+            results.Remove(match);
+            if (!results.Any())
+              results.Insert(idx, new Factor('1')); // when the factor is the last one then insert a one
+
+            // also mark as cancelled in the factorsArr
+            idx = Array.IndexOf(factorsArr, match);
+            factorsArr[idx] = new Factor('1');
+          }
+        }
+      }
+      if (results.Count == 0) results.Add(new Factor('1'));
+      return results;
+    }
+
+    #region term
+    // term can be merged (essentially an ordered list of factors)
+    internal class Term : IComparable<Term> {
+      private readonly SortedList<Factor, int> factors; // factor symbol and the number of occurrences
+
+      public IEnumerable<Factor> Factors {
+        get {
+          return factors.SelectMany(p => Enumerable.Repeat(p.Key, p.Value));
+        }
+      }
+
+      public Term(Factor f) {
+        factors = new SortedList<Factor, int>();
+        factors.Add(f, 1);
+      }
+      public Term(IEnumerable<Factor> factors) {
+        this.factors = new SortedList<Factor, int>();
+        foreach (var f in factors) {
+          if (this.factors.ContainsKey(f)) this.factors[f] += 1;
+          else this.factors.Add(f, 1);
+        }
+      }
+
+      public int CompareTo(Term other) {
+        if (ContainsNonTerminal(Factors) && !ContainsNonTerminal(other.Factors)) {
+          return 1;
+        } else if (!ContainsNonTerminal(Factors) && ContainsNonTerminal(other.Factors)) {
+          return -1;
+        } else {
+          var countComp = Factors.Count().CompareTo(other.Factors.Count());
+          if (countComp != 0) return countComp;
+          foreach (var pair in Factors.Zip(other.Factors, Tuple.Create)) {
+            var fComp = pair.Item1.CompareTo(pair.Item2);
+            if (fComp != 0) return fComp;
+          }
+          return 0;
+        }
+      }
+
+      public override string ToString() {
+
+        return string.Join("*", Factors);
+      }
+      public override bool Equals(object obj) {
+        var other = obj as Term;
+        if (other == null) return false;
+        if (this.Factors.Count() != other.Factors.Count()) return false;
+        if (this.Factors.Zip(other.Factors, Tuple.Create).Any(t => t.Item1 != t.Item2)) return false;
+        return true;
+      }
+      public override int GetHashCode() {
+        var h = 31415;
+        foreach (var v in Factors) {
+          h ^= v.GetHashCode();
+        }
+        return h;
+      }
+    }
+    #endregion
+
+    #region factor
+    // factors is either a single symbol or an inverted expression
+    internal class Factor : IComparable<Factor> {
+      public bool IsSimpleFactor { get { return Expr == null; } }
+      public char Symbol { get { return symbol; } }
+      public Expr Expr { get { return expr; } }
+      public bool IsInverse { get { return inv; } }
+      private readonly char symbol = '\0';
+      private readonly Expr expr;
+      private bool inv;
+
+      public Factor(char f) {
+        this.symbol = f;
+      }
+      public Factor(Expr expr) {
+        this.expr = expr;
+      }
+
+      public void Invert() {
+        this.inv = !inv;
+      }
+      public bool Cancels(Factor other) {
+        if (this.inv == other.inv) return false;
+        if (this.Expr != null && other.Expr == null) return false;
+        if (this.Expr == null && other.Expr != null) return false;
+        if (Expr == null) return this.Symbol.Equals(other.symbol);
+        else return this.Expr.CompareTo(other.Expr) == 0;
+      }
+
+      public int CompareTo(Factor other) {
+        // 1) single symbol factors first
+        // 2) expression factors by expression compare
+        var crit1 = ContainsNonTerminal(this).CompareTo(ContainsNonTerminal(other));
+        if (crit1 != 0) return crit1;
+
+        var crit2 = this.IsInverse.CompareTo(other.IsInverse);
+        if (crit2 != 0) return crit2;
+
+        var crit3 = this.IsSimpleFactor.CompareTo(other.IsSimpleFactor);
+        if (crit3 != 0) return crit3;
+
+        // both are simple or expressions
+        if (IsSimpleFactor) return this.symbol.CompareTo(other.symbol);
+        else return this.Expr.CompareTo(other.Expr);
+      }
+
+      public override string ToString() {
+        var s = Expr == null ? symbol.ToString() : "(" + expr.ToString() + ")";
+        if (IsInverse) {
+          return "/" + s;
+        } else return s;
+      }
+      public override bool Equals(object obj) {
+        var other = obj as Factor;
+        if (other == null) return false;
+        if (IsInverse != other.IsInverse) return false;
+        if (this.symbol != other.symbol) return false;
+        if (this.Expr != other.Expr) return false;
+        return true;
+      }
+      public override int GetHashCode() {
+        var h = 31415;
+        h ^= symbol.GetHashCode();
+        if (Expr != null) h ^= Expr.GetHashCode();
+        return h;
+      }
+    }
+    #endregion
+
+    #region expr
+
+    internal class Expr : IComparable<Expr> {
+      public readonly SortedSet<Term> Terms; // only set for Kind == Expr 
+      //public bool Inverse;
+      public Expr(Term t) {
+        Terms = new SortedSet<Term>();
+        Terms.Add(t);
+      }
+      public Expr(IEnumerable<Term> exprTerms) {
+        Terms = new SortedSet<Term>();
+        foreach (var t in exprTerms) {
+          Terms.Add(t);
+        }
+      }
+
+      public void Merge(Expr other) {
+        this.Terms.UnionWith(other.Terms);
+      }
+
+      public int CompareTo(Expr other) {
+        var sizeComp = this.Terms.Count.CompareTo(other.Terms.Count);
+        if (sizeComp != 0) return sizeComp;
+        // same size => compare terms
+        foreach (var pair in Terms.Zip(other.Terms, Tuple.Create)) {
+          var termComp = pair.Item1.CompareTo(pair.Item2);
+          if (termComp != 0) return termComp;
+        }
+        return 0;
+      }
+
+      public override string ToString() {
+        return string.Join("+", Terms);
+      }
+      public override bool Equals(object obj) {
+        var other = obj as Expr;
+        if (other == null) return false;
+        if (this.Terms.Count() != other.Terms.Count()) return false;
+        return this.Terms.Intersect(other.Terms).Count() == this.Terms.Count;
+      }
+
+      public override int GetHashCode() {
+        var h = 31415;
+        if (Terms != null)
+          foreach (var t in Terms) {
+            h ^= t.GetHashCode();
+          }
+        return h;
+      }
+    }
+    #endregion
+    internal static bool IsNonTerminal(char symb) {
+      return symb >= 'A' && symb <= 'Z';
+    }
+    internal static bool ContainsNonTerminal(IEnumerable<Factor> factors) {
+      return factors.Any(ContainsNonTerminal);
+    }
+    internal static bool ContainsNonTerminal(Factor f) {
+      if (f.Expr == null) return IsNonTerminal(f.Symbol);
+      else return ContainsNonTerminal(f.Expr);
+    }
+
+    private static bool ContainsNonTerminal(Expr expr) {
+      return expr.Terms.Any(ContainsNonTerminal);
+    }
+
+    private static bool ContainsNonTerminal(Term term) {
+      return ContainsNonTerminal(term.Factors);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Extensions.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Extensions.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Extensions.cs	(revision 12290)
@@ -0,0 +1,67 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Common {
+  public static class Extensions {
+
+    public static bool IsAlmost(this double x, double y) {
+      if (double.IsNaN(x) || double.IsNaN(y)) return false;
+      if (double.IsPositiveInfinity(x) && double.IsPositiveInfinity(y)) return true;
+      if (double.IsNegativeInfinity(x) && double.IsNegativeInfinity(y)) return true;
+      return Math.Abs(x - y) < 1.0e-12;
+    }
+
+    public static T SelectRandom<T>(this IEnumerable<T> xs, Random rand) {
+      var n = xs.Count();
+      return xs.ElementAt(rand.Next(n));
+    }
+
+    public static T SampleProportional<T>(this IEnumerable<T> elements, Random random, IEnumerable<double> weights) {
+      double total = weights.Sum();
+
+      var elemEnumerator = elements.GetEnumerator();
+      elemEnumerator.MoveNext();
+      var weightEnumerator = weights.GetEnumerator();
+      weightEnumerator.MoveNext();
+
+      var r = random.NextDouble() * total;
+      var agg = weightEnumerator.Current;
+
+      while (agg < r) {
+        weightEnumerator.MoveNext();
+        elemEnumerator.MoveNext();
+        agg += weightEnumerator.Current;
+      }
+      return elemEnumerator.Current;
+    }
+
+    public static double RSq(IEnumerable<double> xs, IEnumerable<double> ys) {
+      // two pass implementation, but we don't care
+      var meanX = xs.Average();
+      var meanY = ys.Average();
+
+      var s = 0.0;
+      var ssX = 0.0;
+      var ssY = 0.0;
+      var xEnum = xs.GetEnumerator();
+      var yEnum = ys.GetEnumerator();
+      while (xEnum.MoveNext() & yEnum.MoveNext()) {
+        var x = xEnum.Current;
+        var y = yEnum.Current;
+        s += (x - meanX) * (y - meanY);
+        ssX += (x - meanX) * (x - meanX);
+        ssY += (y - meanY) * (y - meanY);
+      }
+      if (xEnum.MoveNext() | yEnum.MoveNext()) throw new ArgumentException("lengths are not equal");
+
+      if (s.IsAlmost(0)) return 0;
+      if (ssX.IsAlmost(0) || ssY.IsAlmost(0)) return 0;
+      return s * s / (ssX * ssY);
+    }
+
+
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/HeuristicLab.Common.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/HeuristicLab.Common.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/HeuristicLab.Common.csproj	(revision 12290)
@@ -0,0 +1,53 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Common</RootNamespace>
+    <AssemblyName>HeuristicLab.Common</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ConsoleEx.cs" />
+    <Compile Include="ExpressionExtender.cs" />
+    <Compile Include="Extensions.cs" />
+    <Compile Include="MostRecentlyUsedCache.cs" />
+    <Compile Include="OnlineMeanAndVarianceEstimator.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="Rand.cs" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/MostRecentlyUsedCache.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/MostRecentlyUsedCache.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/MostRecentlyUsedCache.cs	(revision 12290)
@@ -0,0 +1,45 @@
+﻿using System;
+using System.Collections.Generic;
+using System.ComponentModel;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Common {
+  public class MostRecentlyUsedCache<TKey, TValue> {
+
+    private LinkedList<KeyValuePair<TKey, TValue>> lruList;
+    private Dictionary<TKey, LinkedListNode<KeyValuePair<TKey, TValue>>> dictionary;
+    private int maxCapacity;
+
+    public MostRecentlyUsedCache(int maxCapacity) {
+      this.maxCapacity = maxCapacity;
+      lruList = new LinkedList<KeyValuePair<TKey, TValue>>();
+      dictionary = new Dictionary<TKey, LinkedListNode<KeyValuePair<TKey, TValue>>>(maxCapacity);
+    }
+
+    // drops the least-recently used element if necessary
+    public void Add(TKey key, TValue value) {
+      if (lruList.Count >= maxCapacity) {
+        dictionary.Remove(lruList.Last.Value.Key);
+        lruList.RemoveLast();
+      }
+      lruList.AddFirst(new KeyValuePair<TKey, TValue>(key, value));
+      dictionary.Add(key, lruList.First);
+    }
+
+    public bool TryGetValue(TKey key, out TValue value) {
+      // find matching element
+      LinkedListNode<KeyValuePair<TKey, TValue>> node = null;
+      if (!dictionary.TryGetValue(key, out node)) {
+        value = default(TValue);
+        return false;
+      } else {
+        lruList.Remove(node);
+        lruList.AddFirst(node);
+        value = node.Value.Value;
+        return true;
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/OnlineMeanAndVarianceEstimator.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/OnlineMeanAndVarianceEstimator.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/OnlineMeanAndVarianceEstimator.cs	(revision 12290)
@@ -0,0 +1,35 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  public class OnlineMeanAndVarianceEstimator {
+    public int N { get; private set; }
+    public double Sum { get; private set; }
+    public double Avg { get; private set; }
+    public double Variance { get { return sampleM2 / N; } }
+    private double sampleM2;
+
+    public void UpdateReward(double reward) {
+      N++;
+      Sum += reward;
+
+      var delta = reward - Avg;
+      //Avg += delta / N;
+      // double alpha = 0.01;
+      double alpha = 1.0 / N;
+      Avg = Avg + alpha * (delta);
+      sampleM2 += delta * (reward - Avg);
+    }
+
+    public void Reset() {
+      Sum = 0.0;
+      Avg = 0.0;
+      N = 0;
+      sampleM2 = 0.0;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Common")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("FH-Hagenberg")]
+[assembly: AssemblyProduct("HeuristicLab.Common")]
+[assembly: AssemblyCopyright("Copyright © FH-Hagenberg 2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("756c28a7-4012-4f23-927b-a6baa8eefa40")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Rand.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Rand.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Common/Rand.cs	(revision 12290)
@@ -0,0 +1,69 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Common {
+  // code from Tom Minka's lightspeed toolbox (random.c)
+  // original copyright: 
+  // Written by Tom Minka
+  // (c) Microsoft Corporation. All rights reserved.
+  public static class Rand {
+
+    /* Returns a sample from Normal(0,1) */
+
+    public static double RandNormal(Random random) {
+      double x, y, radius;
+      /* Generate a random point inside the unit circle */
+      do {
+        x = 2 * random.NextDouble() - 1;
+        y = 2 * random.NextDouble() - 1;
+        radius = (x * x) + (y * y);
+      } while ((radius >= 1.0) || (radius == 0.0));
+      /* Box-Muller formula */
+      radius = Math.Sqrt(-2 * Math.Log(radius) / radius);
+      x *= radius;
+      return x;
+    }
+
+
+    /* Returns a sample from Gamma(a, 1).
+     * For Gamma(a,b), scale the result by b.
+     */
+    public static double GammaRand(Random random, double a) {
+      /* Algorithm:
+       * G. Marsaglia and W.W. Tsang, A simple method for generating gamma
+       * variables, ACM Transactions on Mathematical Software, Vol. 26, No. 3,
+       * Pages 363-372, September, 2000.
+       * http://portal.acm.org/citation.cfm?id=358414
+       */
+      double boost, d, c, v;
+      if (a < 1) {
+        /* boost using Marsaglia's (1961) method: gam(a) = gam(a+1)*U^(1/a) */
+        boost = Math.Exp(Math.Log(random.NextDouble()) / a);
+        a++;
+      } else boost = 1;
+      d = a - 1.0 / 3; c = 1.0 / Math.Sqrt(9 * d);
+      while (true) {
+        double x, u;
+        do {
+          x = RandNormal(random);
+          v = 1 + c * x;
+        } while (v <= 0);
+        v = v * v * v;
+        x = x * x;
+        u = random.NextDouble();
+        if ((u < 1 - .0331 * x * x) ||
+           (Math.Log(u) < 0.5 * x + d * (1 - v + Math.Log(v)))) break;
+      }
+      return (boost * d * v);
+    }
+
+    /* Returns a sample from Beta(a,b) */
+    public static double BetaRand(Random random, double a, double b) {
+      double g = GammaRand(random, a);
+      return g / (g + GammaRand(random, b));
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/BernoulliModel.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/BernoulliModel.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/BernoulliModel.cs	(revision 12290)
@@ -0,0 +1,51 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.Models {
+  public class BernoulliModel : IModel {
+
+    private int success;
+    private int failure;
+
+    // parameters of beta prior distribution
+    private readonly double alpha;
+    private readonly double beta;
+
+    public BernoulliModel(double alpha = 1.0, double beta = 1.0) {
+      this.alpha = alpha;
+      this.beta = beta;
+    }
+
+    public double Sample(Random random) {
+      // sample bernoulli mean from beta prior
+      return Rand.BetaRand(random, success + alpha, failure + beta);
+    }
+
+    public void Update(double reward) {
+      // Debug.Assert(reward.IsAlmost(1.0) || reward.IsAlmost(0.0));
+      if (reward > 0) {
+        success++;
+      } else {
+        failure++;
+      }
+    }
+
+    public void Reset() {
+      success = 0;
+      failure = 0;
+    }
+
+    public object Clone() {
+      return new BernoulliModel() { failure = this.failure, success = this.success };
+    }
+
+    public override string ToString() {
+      return string.Format("Bernoulli with Beta prior: mu={0:F2}", (success + alpha) / (success + alpha + failure + beta));
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/GaussianMixtureModel.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/GaussianMixtureModel.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/GaussianMixtureModel.cs	(revision 12290)
@@ -0,0 +1,99 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.Models {
+  public class GaussianMixtureModel : IModel {
+    private double[] componentMeans;
+    private double[] componentVars;
+    private double[] componentProbs;
+    private readonly List<double> allRewards = new List<double>();
+
+    private int numComponents;
+
+    public GaussianMixtureModel(int nComponents = 5) {
+      this.numComponents = nComponents;
+
+      Reset();
+    }
+
+
+    public double Sample(Random random) {
+      var k = Enumerable.Range(0, numComponents).SampleProportional(random, componentProbs);
+      return alglib.invnormaldistribution(random.NextDouble()) * Math.Sqrt(componentVars[k]) + componentMeans[k];
+    }
+
+    public void Update(double reward) {
+      allRewards.Add(reward);
+      throw new NotSupportedException("this does not yet work");
+      if (allRewards.Count < 1000 && allRewards.Count % 10 == 0) {
+        // see http://www.cs.toronto.edu/~mackay/itprnn/ps/302.320.pdf Algorithm 22.2 soft k-means
+        Reset();
+        for (int i = 0; i < 20; i++) {
+          var responsibilities = allRewards.Select(r => CalcResponsibility(r)).ToArray();
+
+
+          var sumWeightedRewards = new double[numComponents];
+          var sumResponsibilities = new double[numComponents];
+          foreach (var p in allRewards.Zip(responsibilities, Tuple.Create)) {
+            for (int k = 0; k < numComponents; k++) {
+              sumWeightedRewards[k] += p.Item2[k] * p.Item1;
+              sumResponsibilities[k] += p.Item2[k];
+            }
+          }
+          for (int k = 0; k < numComponents; k++) {
+            componentMeans[k] = sumWeightedRewards[k] / sumResponsibilities[k];
+          }
+
+          sumWeightedRewards = new double[numComponents];
+          foreach (var p in allRewards.Zip(responsibilities, Tuple.Create)) {
+            for (int k = 0; k < numComponents; k++) {
+              sumWeightedRewards[k] += p.Item2[k] * Math.Pow(p.Item1 - componentMeans[k], 2);
+            }
+          }
+          for (int k = 0; k < numComponents; k++) {
+            componentVars[k] = sumWeightedRewards[k] / sumResponsibilities[k];
+            componentProbs[k] = sumResponsibilities[k] / sumResponsibilities.Sum();
+          }
+        }
+      }
+    }
+
+    private double[] CalcResponsibility(double r) {
+      var res = new double[numComponents];
+      for (int k = 0; k < numComponents; k++) {
+        componentVars[k] = Math.Max(componentVars[k], 0.001);
+        res[k] = componentProbs[k] * alglib.normaldistribution((r - componentMeans[k]) / Math.Sqrt(componentVars[k]));
+        res[k] = Math.Max(res[k], 0.0001);
+      }
+      var sum = res.Sum();
+      for (int k = 0; k < numComponents; k++) {
+        res[k] /= sum;
+      }
+      return res;
+    }
+
+    public void Disable() {
+      Array.Clear(componentMeans, 0, numComponents);
+      for (int i = 0; i < numComponents; i++)
+        componentVars[i] = 0.0;
+    }
+
+    public object Clone() {
+      return new GaussianMixtureModel(numComponents);
+    }
+
+    public void Reset() {
+      var rand = new Random();
+      this.componentProbs = Enumerable.Range(0, numComponents).Select((_) => rand.NextDouble()).ToArray();
+      var sum = componentProbs.Sum();
+      for (int i = 0; i < componentProbs.Length; i++) componentProbs[i] /= sum;
+      this.componentMeans = Enumerable.Range(0, numComponents).Select((_) => Rand.RandNormal(rand)).ToArray();
+      this.componentVars = Enumerable.Range(0, numComponents).Select((_) => 0.01).ToArray();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/GaussianModel.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/GaussianModel.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/GaussianModel.cs	(revision 12290)
@@ -0,0 +1,145 @@
+﻿using System;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.Models {
+  // bayesian estimation of a Gaussian with 
+  // 1) unknown mean and known variance
+  // 2) unknown mean and unknown variance
+  public class GaussianModel : IModel {
+
+    private OnlineMeanAndVarianceEstimator estimator = new OnlineMeanAndVarianceEstimator();
+
+    // parameters of Gaussian prior for mean
+    private readonly double meanPriorMu;
+    private readonly double meanPriorVariance;
+
+    private readonly bool knownVariance;
+    private readonly double rewardVariance = 0.1; // assumed know reward variance
+
+    // parameters of Gamma prior for precision (= inverse variance)
+    private readonly int precisionPriorAlpha;
+    private readonly double precisionPriorBeta;
+
+    // non-informative prior
+    private const double priorK = 1.0;
+
+    // this constructor assumes the variance is known
+    public GaussianModel(double meanPriorMu, double meanPriorVariance, double rewardVariance = 0.1) {
+      this.meanPriorMu = meanPriorMu;
+      this.meanPriorVariance = meanPriorVariance;
+
+      this.knownVariance = true;
+      this.rewardVariance = rewardVariance;
+    }
+
+    // this constructor assumes the variance is also unknown 
+    // uses Murphy 2007: Conjugate Bayesian analysis of the Gaussian distribution equation 85 - 89
+    public GaussianModel(double meanPriorMu, double meanPriorVariance, int precisionPriorAlpha, double precisionPriorBeta) {
+      this.meanPriorMu = meanPriorMu;
+      this.meanPriorVariance = meanPriorVariance;
+      
+      this.knownVariance = false;
+      this.precisionPriorAlpha = precisionPriorAlpha;
+      this.precisionPriorBeta = precisionPriorBeta;
+    }
+
+
+    public double Sample(Random random) {
+      if (knownVariance) {
+        return SampleExpectedRewardKnownVariance(random);
+      } else {
+        return SampleExpectedRewardUnknownVariance(random);
+      }
+    }
+
+    private double SampleExpectedRewardKnownVariance(Random random) {
+      // expected values for reward
+      // calculate posterior mean and variance (for mean reward)
+
+      // see Murphy 2007: Conjugate Bayesian analysis of the Gaussian distribution (http://www.cs.ubc.ca/~murphyk/Papers/bayesGauss.pdf)
+      var posteriorMeanVariance = 1.0 / (estimator.N / rewardVariance + 1.0 / meanPriorVariance);
+      var posteriorMeanMean = posteriorMeanVariance * (meanPriorMu / meanPriorVariance + estimator.Sum / rewardVariance);
+
+      // sample a mean from the posterior 
+      var posteriorMeanSample = Rand.RandNormal(random) * Math.Sqrt(posteriorMeanVariance) + posteriorMeanMean;
+      // theta already represents the expected reward value => nothing else to do
+      return posteriorMeanSample;
+
+      // return 0.99-quantile value
+      //return alglib.invnormaldistribution(0.99) * Math.Sqrt(rewardVariance + posteriorMeanVariance) + posteriorMeanMean;
+    }
+
+    // see Murphy 2007: Conjugate Bayesian analysis of the Gaussian distribution page 6 onwards (http://www.cs.ubc.ca/~murphyk/Papers/bayesGauss.pdf)
+    private double SampleExpectedRewardUnknownVariance(Random random) {
+
+      var posteriorMean = (priorK * meanPriorMu + estimator.Sum) / (priorK + estimator.N);
+      var posteriorK = priorK + estimator.N;
+      var posteriorAlpha = precisionPriorAlpha + estimator.N / 2.0;
+      double posteriorBeta;
+      if (estimator.N > 0) {
+        posteriorBeta = precisionPriorBeta + 0.5 * estimator.N * estimator.Variance + priorK * estimator.N * Math.Pow(estimator.Avg - meanPriorMu, 2) / (2.0 * (priorK + estimator.N));
+      } else {
+        posteriorBeta = precisionPriorBeta;
+      }
+
+
+      // sample from the posterior marginal for mu (expected value) equ. 91
+      // p(µ|D) = T2αn (µ| µn, βn/(αnκn)) 
+
+      var t2alpha = alglib.invstudenttdistribution((int)(2 * posteriorAlpha), random.NextDouble());
+
+      var theta = t2alpha * posteriorBeta / (posteriorAlpha * posteriorK) + posteriorMean;
+      return theta;
+
+
+      /*
+       * value function : 0.99-quantile
+      // sample posterior mean and posterior variance independently
+      var sampledPrec = Rand.GammaRand(random, posteriorAlpha) * posteriorBeta;
+      var t2alpha = alglib.invstudenttdistribution((int)(2 * posteriorAlpha), random.NextDouble());
+      var sampledMean = t2alpha * posteriorBeta / (posteriorAlpha * posteriorK) + posteriorMean;
+
+      return alglib.invnormaldistribution(0.99) / Math.Sqrt(sampledPrec) + sampledMean;
+       */
+    }
+
+
+    public void Update(double reward) {
+      estimator.UpdateReward(reward);
+    }
+
+    public void Reset() {
+      estimator.Reset();
+    }
+
+    public object Clone() {
+      if (knownVariance)
+        return new GaussianModel(meanPriorMu, meanPriorVariance, rewardVariance);
+      else
+        return new GaussianModel(meanPriorMu, meanPriorVariance, precisionPriorAlpha, precisionPriorBeta);
+    }
+
+    public override string ToString() {
+      if (knownVariance) {
+        var posteriorMeanVariance = 1.0 / (estimator.N / rewardVariance + 1.0 / meanPriorVariance);
+        var posteriorMeanMean = posteriorMeanVariance * (meanPriorMu / meanPriorVariance + estimator.Sum / rewardVariance);
+        return string.Format("Gaussian(mu, var=0.1), mu ~ Gaussian(mu'={0:F3}, var'={1:F3})", posteriorMeanMean, posteriorMeanVariance);
+      } else {
+        var posteriorMean = (priorK * meanPriorMu + estimator.Sum) / (priorK + estimator.N);
+        var posteriorK = priorK + estimator.N;
+        var posteriorAlpha = precisionPriorAlpha + estimator.N / 2.0;
+        double posteriorBeta;
+        if (estimator.N > 0) {
+          posteriorBeta = precisionPriorBeta + 0.5 * estimator.N * estimator.Variance + priorK * estimator.N * Math.Pow(estimator.Avg - meanPriorMu, 2) / (2.0 * (priorK + estimator.N));
+        } else {
+          posteriorBeta = precisionPriorBeta;
+        }
+        var nu = (int)(2 * posteriorAlpha);
+        var meanVariance = posteriorBeta / (posteriorAlpha * posteriorK) * (nu / (double)(nu - 2));
+        return string.Format("Gaussian(mu, var), mu ~ T{0}(mu'={1:F3}, var'={2:F3}), 1.0/var ~ Gamma(mu={3:F3}, var={4:F3})",
+          nu, posteriorMean, meanVariance,
+          posteriorAlpha / posteriorBeta, posteriorAlpha / (posteriorBeta * posteriorBeta));
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/HeuristicLab.Distributions.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/HeuristicLab.Distributions.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/HeuristicLab.Distributions.csproj	(revision 12290)
@@ -0,0 +1,62 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Distributions</RootNamespace>
+    <AssemblyName>HeuristicLab.Distributions</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="ALGLIB-3.7.0, Version=3.7.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\ALGLIB-3.7.0.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="BernoulliModel.cs" />
+    <Compile Include="GaussianMixtureModel.cs" />
+    <Compile Include="GaussianModel.cs" />
+    <Compile Include="IModel.cs" />
+    <Compile Include="LogitNormalModel.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/IModel.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/IModel.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/IModel.cs	(revision 12290)
@@ -0,0 +1,14 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  // represents a model for the reward distribution (of an action given a state)
+  public interface IModel : ICloneable {
+    double Sample(Random random);
+    void Update(double reward);
+    void Reset();
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/LogitNormalModel.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/LogitNormalModel.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/LogitNormalModel.cs	(revision 12290)
@@ -0,0 +1,35 @@
+﻿using System;
+using System.Runtime.InteropServices;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits.Models {
+  public class LogitNormalModel : IModel {
+
+    private readonly GaussianModel gaussian;
+
+    // this constructor assumes the variance is known
+    public LogitNormalModel() { gaussian = new GaussianModel(0, 1000, 1, 1); }
+
+
+
+    public double Sample(Random random) {
+      return 1.0 / (1 + Math.Exp(-gaussian.Sample(random)));
+    }
+
+    public void Update(double reward) {
+      gaussian.Update(Math.Log(reward / (1 - reward)));
+    }
+
+    public void Reset() {
+      gaussian.Reset();
+    }
+
+    public object Clone() {
+      return new LogitNormalModel();
+    }
+
+    public override string ToString() {
+      return string.Empty;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Distributions/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Distributions")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("FH-Hagenberg")]
+[assembly: AssemblyProduct("HeuristicLab.Distributions")]
+[assembly: AssemblyCopyright("Copyright © FH-Hagenberg 2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("d23ca007-4842-4ccd-89b3-21b6e9df0273")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/BanditHelper.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/BanditHelper.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/BanditHelper.cs	(revision 12290)
@@ -0,0 +1,37 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  class BanditHelper {
+    public static void SampleArms(Random random, IBandit bandit, int nSamples, out double expRewardEst, out int bestArmForExpReward, out int bestArmForMaxReward) {
+      bestArmForExpReward = 0;
+      bestArmForMaxReward = 0;
+      expRewardEst = 0.0;
+      var bestSumReward = 0.0;
+      var bestUpperPercCount = 0;
+      for (int a = 0; a < bandit.NumArms; a++) {
+        var sumReward = 0.0;
+        var upperPercCount = 0;
+
+        for (int i = 0; i < nSamples; i++) {
+          var reward = bandit.Pull(a);
+          sumReward += reward;
+          if (reward >= 0.95) upperPercCount++;
+        }
+
+        if (sumReward > bestSumReward) {
+          bestSumReward = sumReward;
+          bestArmForExpReward = a;
+          expRewardEst = bestSumReward / nSamples;
+        }
+        if (upperPercCount > bestUpperPercCount) {
+          bestUpperPercCount = upperPercCount;
+          bestArmForMaxReward = a;
+        }
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/BernoulliBandit.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/BernoulliBandit.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/BernoulliBandit.cs	(revision 12290)
@@ -0,0 +1,38 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  public class BernoulliBandit : IBandit {
+    public int NumArms { get; private set; }
+    public double OptimalExpectedReward { get; private set; } // reward of the best arm, for calculating regret
+    public int OptimalExpectedRewardArm { get; private set; }
+    // the arm with highest expected reward also has the highest probability of return a reward of 1.0
+    public int OptimalMaximalRewardArm { get { return OptimalExpectedRewardArm; } }
+
+    private readonly Random random;
+    private readonly double[] expReward;
+    public BernoulliBandit(Random random, int nArms) {
+      this.random = random;
+      this.NumArms = nArms;
+      // expected reward of arms is iid and uniformly distributed 
+      expReward = new double[nArms];
+      OptimalExpectedReward = double.NegativeInfinity;
+      for (int i = 0; i < nArms; i++) {
+        expReward[i] = random.NextDouble();
+        if (expReward[i] > OptimalExpectedReward) {
+          OptimalExpectedReward = expReward[i];
+          OptimalExpectedRewardArm = i;
+        }
+      }
+    }
+
+    // pulling an arm results in a bernoulli distributed reward 
+    // with mean expReward[i]
+    public double Pull(int arm) {
+      return random.NextDouble() <= expReward[arm] ? 1 : 0;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/GaussianBandit.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/GaussianBandit.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/GaussianBandit.cs	(revision 12290)
@@ -0,0 +1,49 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  public class GaussianBandit : IBandit {
+    public int NumArms { get; private set; }
+    public double OptimalExpectedReward { get; private set; } // reward of the best arm, for calculating regret
+    public int OptimalExpectedRewardArm { get; private set; }
+    public int OptimalMaximalRewardArm { get; private set; }
+
+    private readonly Random random;
+    private readonly double[] exp;
+    private readonly double[] stdDev;
+    public GaussianBandit(Random random, int nArms) {
+      this.random = random;
+      this.NumArms = nArms;
+      // expected reward of arms is iid and uniformly distributed 
+      exp = new double[nArms];
+      stdDev = new double[nArms];
+      OptimalExpectedReward = double.NegativeInfinity;
+      var bestQ = double.NegativeInfinity;
+      for (int i = 0; i < nArms; i++) {
+        exp[i] = Rand.RandNormal(random);  // exp values for arms is N(0,1) distributed
+        stdDev[i] = 1.0 / Rand.GammaRand(random, 1); // variance is inv-gamma distributed 
+        if (exp[i] > OptimalExpectedReward) {
+          OptimalExpectedReward = exp[i];
+          OptimalExpectedRewardArm = i;
+        }
+        var q = alglib.invnormaldistribution(0.99) * stdDev[i] + exp[i];
+        if (q > bestQ) {
+          bestQ = q;
+          OptimalMaximalRewardArm = i;
+        }
+      }
+    }
+
+    // pulling an arm results in a truncated normally distributed reward 
+    // with mean expReward[i] and std.dev 0.1
+    public double Pull(int arm) {
+      var z = Rand.RandNormal(random);
+      var x = z * stdDev[arm] + exp[arm];
+      return x;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/GaussianMixtureBandit.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/GaussianMixtureBandit.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/GaussianMixtureBandit.cs	(revision 12290)
@@ -0,0 +1,55 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  // uses a gaussian mixture reward distribution for each arm
+  public class GaussianMixtureBandit : IBandit {
+    public int NumArms { get; private set; }
+    public double OptimalExpectedReward { get; private set; } // reward of the best arm, for calculating regret
+    public int OptimalExpectedRewardArm { get; private set; }
+    public int OptimalMaximalRewardArm { get; private set; }
+
+    private readonly Random random;
+    private readonly double[] expReward; // for each component components
+    private readonly double[][] componentProb; // arms x components
+    public GaussianMixtureBandit(Random random, int nArms) {
+      this.random = random;
+      this.NumArms = nArms;
+      var numComponents = 0;
+      expReward = new double[] { 0.1, 0.3, 0.5, 0.7, 0.9 };
+      componentProb = new double[nArms][];
+      OptimalExpectedReward = double.NegativeInfinity;
+      // decide on optimal arm
+      OptimalMaximalRewardArm = random.Next(NumArms);
+      OptimalExpectedRewardArm = OptimalMaximalRewardArm;
+      for (int i = 0; i < nArms; i++) {
+        componentProb[i] = new double[numComponents];
+        if (i == OptimalMaximalRewardArm) {
+          componentProb[i] = new double[] { 0.24, 0.24, 0.24, 0.24, 0.04 };
+        } else {
+          componentProb[i] = new double[] { 0.25, 0.25, 0.25, 0.25, 0 };
+        }
+      }
+
+      OptimalExpectedReward = Enumerable.Range(0, 100000).Select(_ => Pull(OptimalExpectedRewardArm)).Average();
+    }
+
+    // std.dev = 0.1 
+    // and truncation to the interval [0..1]
+    public double Pull(int arm) {
+      double x = 0;
+      do {
+        var k = Enumerable.Range(0, componentProb[arm].Length).SampleProportional(random, componentProb[arm]);
+
+        var z = Rand.RandNormal(random);
+        x = z * 0.1 + expReward[k];
+      }
+      while (x < 0 || x > 1);
+      return x;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/HeuristicLab.Problems.Bandits.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/HeuristicLab.Problems.Bandits.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/HeuristicLab.Problems.Bandits.csproj	(revision 12290)
@@ -0,0 +1,63 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{E8138227-0C64-4E85-B676-21D3B97F254F}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Problems.Bandits</RootNamespace>
+    <AssemblyName>HeuristicLab.Problems.Bandits</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="ALGLIB-3.7.0, Version=3.7.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\ALGLIB-3.7.0.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="BanditHelper.cs" />
+    <Compile Include="BernoulliBandit.cs" />
+    <Compile Include="GaussianBandit.cs" />
+    <Compile Include="GaussianMixtureBandit.cs" />
+    <Compile Include="IBandit.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="TruncatedNormalBandit.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/IBandit.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/IBandit.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/IBandit.cs	(revision 12290)
@@ -0,0 +1,16 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  public interface IBandit {
+    int NumArms { get; }
+    double OptimalExpectedReward { get; } // expected reward of the best arm, for calculating regret
+    int OptimalExpectedRewardArm { get; } // arm which is optimal for optimization of expected reward
+    int OptimalMaximalRewardArm { get; } // arm which is optimal for optimization of maximal reward
+
+    double Pull(int arm); // pulling an arm returns a regret
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Problems.Bandits")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("FH-Hagenberg")]
+[assembly: AssemblyProduct("HeuristicLab.Problems.Bandits")]
+[assembly: AssemblyCopyright("Copyright © FH-Hagenberg 2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("54229c20-7557-41e9-b81a-e1fde89f3ab0")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/TruncatedNormalBandit.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/TruncatedNormalBandit.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.Bandits/TruncatedNormalBandit.cs	(revision 12290)
@@ -0,0 +1,45 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Algorithms.Bandits {
+  public class TruncatedNormalBandit : IBandit {
+    public int NumArms { get; private set; }
+    public double OptimalExpectedReward { get; private set; } // reward of the best arm, for calculating regret
+    public int OptimalExpectedRewardArm { get; private set; }
+    // the arm with highest expected reward also has the highest probability of return a reward of 1.0
+    public int OptimalMaximalRewardArm { get { return OptimalExpectedRewardArm; } }
+
+    private readonly Random random;
+    private readonly double[] expReward;
+    public TruncatedNormalBandit(Random random, int nArms) {
+      this.random = random;
+      this.NumArms = nArms;
+      // expected reward of arms is iid and uniformly distributed 
+      expReward = new double[nArms];
+      OptimalExpectedReward = double.NegativeInfinity;
+      for (int i = 0; i < nArms; i++) {
+        expReward[i] = random.NextDouble() * 0.7;
+        if (expReward[i] > OptimalExpectedReward) {
+          OptimalExpectedReward = expReward[i];
+          OptimalExpectedRewardArm = i;
+        }
+      }
+    }
+
+    // pulling an arm results in a truncated normally distributed reward 
+    // with mean expReward[i] and std.dev 0.1
+    public double Pull(int arm) {
+      double x = 0;
+      do {
+        var z = Rand.RandNormal(random);
+        x = z * 0.1 + expReward[arm];
+      }
+      while (x < 0 || x > 1);
+      return x;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/ExpressionCompiler.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/ExpressionCompiler.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/ExpressionCompiler.cs	(revision 12290)
@@ -0,0 +1,158 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics.Eventing.Reader;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.Remoting.Messaging;
+using System.Text;
+using System.Threading.Tasks;
+using AutoDiff;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class ExpressionCompiler {
+    private string sentence;
+    private int syIdx;
+    // compiles sentences from L(G(Expr)) using AutoDiff
+    // does not compile to IL it is only necessary to calculate gradients for parameter optimization
+    // Expr -> Term { ('+' | '-' | '^' ) Term } 
+    // Term -> Fact { ('*' | '%' | '/') Fact }
+    // Fact -> '!' Expr | '(' Expr ')' | Var | const | one
+    // Var -> 'a'..'z'
+    // const -> '0' .. '9'
+    // one -> |    // pipe symbol for constant one instead of ERC 1
+
+    // constants are completely ignored, instead we introduce a multiplicative constant factor for each term and an additive constant term for each expression
+
+    // for boolean problems the symbol * representes the AND operator, the symbol + represents the OR operator, ! = NOT, ^ = XOR
+
+    public void Compile(string sentence, out Term f, Variable[] variables, out Variable[] constants) {
+      InitLex(sentence);
+      var constantsList = new List<Variable>();
+      f = Expr(variables, constantsList);
+      constants = constantsList.ToArray();
+    }
+
+    private void InitLex(string sentence) {
+      this.sentence = sentence;
+      this.syIdx = 0;
+    }
+
+    private char CurSy() {
+      if (syIdx >= sentence.Length) return '\0';
+      return sentence[syIdx];
+    }
+    private void NewSy() {
+      if (syIdx < sentence.Length) syIdx++;
+    }
+
+    // helper for xor
+    private Term Not(Term x) {
+      return 1.0 - x;
+    }
+
+    private Term Expr(IList<Variable> variables, IList<Variable> constants) {
+      var terms = new List<Term>();
+      terms.Add(Term(variables, constants));
+      var curSy = CurSy();
+      while (curSy == '+' || curSy == '-' || curSy == '^') {
+        if (curSy == '+') {
+          NewSy();
+          terms.Add(Term(variables, constants));
+        } else if (curSy == '-') {
+          NewSy();
+          terms.Add(-Term(variables, constants)); // minus
+        } else {
+          NewSy();
+          throw new NotImplementedException();
+          // var e = Expr(variables, constants);
+          // r = Not(r) * e + r * Not(e); // xor = (!x AND y) OR (x AND !y)
+        }
+        curSy = CurSy();
+      }
+
+      // at this point we might have multiple constants in the list of terms but we only need one of them
+      var firstConstant = terms.OfType<Variable>().FirstOrDefault(); // if a term is only a variable then it must be a constant (otherwise (for real variables) it would be a product of two constants)
+      if (firstConstant == null) {
+        // no constant found => add a constant offset for each full expression
+        var offset = new Variable();
+        constants.Add(offset);
+        terms.Add(offset);
+      } else {
+        // there is already a constant => remove all others if there are more
+        var otherConsts = terms.OfType<Variable>().Where(t => t != firstConstant).ToArray();
+        foreach (var otherConst in otherConsts) {
+          terms.Remove(otherConst);
+          constants.Remove(otherConst);
+        }
+      }
+      return TermBuilder.Sum(terms);
+    }
+
+    private Term Term(IList<Variable> variables, IList<Variable> constants) {
+      var factors = new List<Term>();
+      var f = Fact(variables, constants);
+      if (f != null) factors.Add(f);
+      var curSy = CurSy();
+      while (curSy == '*' || curSy == '%') { // division and protected division symbols are handled in the same way
+        if (curSy == '*') {
+          NewSy();
+          f = Fact(variables, constants); // fact might return null (for constants)
+          if (f != null) factors.Add(f);
+        } else {
+          NewSy();
+          f = Fact(variables, constants);
+          if (f != null) factors.Add(1.0 / f); // cannot use protected division here
+        }
+        curSy = CurSy();
+      }
+
+      // generate a constant factor (scale) for each full term
+      // since we ignore constants in the string it could be possible that factors is still empty at this point
+      var scale = new Variable();
+      constants.Add(scale);
+      factors.Add(scale);
+
+      if (factors.Count == 1) return factors[0];
+      else if (factors.Count == 2) return factors[0] * factors[1];
+      else return TermBuilder.Product(factors[0], factors[1], factors.Skip(2).ToArray());
+    }
+
+    private Term Fact(IList<Variable> variables, IList<Variable> constants) {
+      Term r = null;
+      var curSy = CurSy();
+      if (curSy == '!') {
+        NewSy();
+        r = Not(Expr(variables, constants));
+      } else if (curSy == '(') {
+        NewSy();
+        r = Expr(variables, constants);
+        if (CurSy() != ')') throw new ArgumentException();
+        NewSy();
+      } else if (curSy >= 'a' && curSy <= 'z') {
+        var varIdx = (byte)curSy - (byte)'a';
+        if (varIdx >= variables.Count) throw new ArgumentException();
+        r = variables[varIdx];
+        NewSy();
+      } else if (curSy == '/') {
+        // /-symbol used in the expressionextender to represent inverse (1/x). 
+        // this is necessary because we also use symbols 0..9 as indices for ERCs
+        NewSy();
+        r = 1.0 / Fact(variables, constants);
+      } else if (curSy >= '0' && curSy <= '9') {
+        int o = (byte)curSy - (byte)'0';
+        //int o = Convert.ToByte(CurSy()) - Convert.ToByte('a');
+        if (o < 0 || o >= 10) throw new ArgumentException();
+
+        // completely ignore existing constants
+
+        // var newConst = new Variable();
+        // constants.Add(newConst);
+        // r = newConst;
+        NewSy();
+      } else throw new ArgumentException();
+      return r;
+    }
+
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/HeuristicLab.Problems.GrammaticalOptimization.SymbReg.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/HeuristicLab.Problems.GrammaticalOptimization.SymbReg.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/HeuristicLab.Problems.GrammaticalOptimization.SymbReg.csproj	(revision 12290)
@@ -0,0 +1,94 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{17A7A380-86CE-482D-8D22-CBD70CC97F0D}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Problems.GrammaticalOptimization.SymbReg</RootNamespace>
+    <AssemblyName>HeuristicLab.Problems.GrammaticalOptimization.SymbReg</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="ALGLIB-3.7.0, Version=3.7.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\ALGLIB-3.7.0.dll</HintPath>
+    </Reference>
+    <Reference Include="AutoDiff-1.0, Version=1.0.0.14388, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\AutoDiff-1.0.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Common-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Core-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Core-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Data-3.3">
+      <HintPath>..\HeuristicLab\HeuristicLab.Data-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.DataAnalysis-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Problems.DataAnalysis-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.Instances-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Problems.Instances-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.Instances.DataAnalysis-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Problems.Instances.DataAnalysis-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="ExpressionCompiler.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="SymbolicRegressionProblem.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3a2fbbcb-f9df-4970-87f3-f13337d941ad}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.csproj">
+      <Project>{cb9dccf6-667e-4a13-b82d-dbd6b45a045e}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Problems.GrammaticalOptimization.SymbReg")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("FH-Hagenberg")]
+[assembly: AssemblyProduct("HeuristicLab.Problems.GrammaticalOptimization.SymbReg")]
+[assembly: AssemblyCopyright("Copyright © FH-Hagenberg 2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("005ac6c4-c6be-4db5-8d05-09184878eebc")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/SymbolicRegressionProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/SymbolicRegressionProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization.SymbReg/SymbolicRegressionProblem.cs	(revision 12290)
@@ -0,0 +1,340 @@
+﻿using System;
+using System.CodeDom.Compiler;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Security;
+using System.Security.AccessControl;
+using System.Security.Authentication.ExtendedProtection.Configuration;
+using System.Text;
+using AutoDiff;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Common;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.Instances;
+using HeuristicLab.Problems.Instances.DataAnalysis;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.SymbReg {
+  // provides bridge to HL regression problem instances
+  public class SymbolicRegressionProblem : ISymbolicExpressionTreeProblem {
+
+    // C represents Koza-style ERC (the symbol is the index of the ERC), the values are initialized below
+    private const string grammarString = @"
+        G(E):
+        E -> V | C | V+E | V-E | V*E | V%E | (E) | C+E | C-E | C*E | C%E 
+        C -> 0..9
+        V -> <variables>
+        ";
+
+    // when using constant opt optimal constants are generated in the evaluation step so we don't need them in the grammar
+    private const string grammarConstOptString = @"
+        G(E):
+        E -> V | V+E | V*E | V%E | (E) 
+        V -> <variables>
+        ";
+
+    // S .. Sum (+), N .. Neg. sum (-), P .. Product (*), D .. Division (%)
+    private const string treeGrammarString = @"
+        G(E):
+        E -> V | C | S | N | P | D 
+        S -> EE | EEE
+        N -> EE | EEE
+        P -> EE | EEE
+        D -> EE
+        C -> 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9
+        V -> <variables>
+        ";
+    private const string treeGrammarConstOptString = @"
+        G(E):
+        E -> V | S | P | D 
+        S -> EE | EEE
+        P -> EE | EEE
+        D -> EE
+        V -> <variables>
+        ";
+
+    // when we use constants optimization we can completely ignore all constants by a simple strategy:
+    // introduce a constant factor for each complete term
+    // introduce a constant offset for each complete expression (including expressions in brackets)
+    // e.g. 1*(2*a + b - 3 + 4) is the same as c0*a + c1*b + c2
+
+    private readonly IGrammar grammar;
+
+    private readonly int N;
+    private readonly double[,] x;
+    private readonly double[] y;
+    private readonly int d;
+    private readonly double[] erc;
+    private readonly bool useConstantOpt;
+    public string Name { get; private set; }
+    private Random random;
+    private double lambda;
+
+
+    // lambda should be tuned using CV
+    public SymbolicRegressionProblem(Random random, string partOfName, double lambda = 1.0, bool useConstantOpt = true) {
+      var instanceProviders = new RegressionInstanceProvider[] 
+      {new RegressionRealWorldInstanceProvider(),
+        new HeuristicLab.Problems.Instances.DataAnalysis.VariousInstanceProvider(),
+        new KeijzerInstanceProvider(), 
+        new VladislavlevaInstanceProvider(), 
+        new NguyenInstanceProvider(), 
+        new KornsInstanceProvider(), 
+    };
+      var instanceProvider = instanceProviders.FirstOrDefault(prov => prov.GetDataDescriptors().Any(dd => dd.Name.Contains(partOfName)));
+      IRegressionProblemData problemData = null;
+      if (instanceProvider != null) {
+        var dds = instanceProvider.GetDataDescriptors();
+        problemData = instanceProvider.LoadData(dds.Single(ds => ds.Name.Contains(partOfName)));
+
+      } else if (File.Exists(partOfName)) {
+        // check if it is a file
+        var prov = new RegressionCSVInstanceProvider();
+        problemData = prov.ImportData(partOfName);
+        problemData.TrainingPartition.Start = 0;
+        problemData.TrainingPartition.End = problemData.Dataset.Rows;
+        // no test partition
+        problemData.TestPartition.Start = problemData.Dataset.Rows;
+        problemData.TestPartition.End = problemData.Dataset.Rows;
+      } else {
+        throw new ArgumentException("instance not found");
+      }
+
+      this.useConstantOpt = useConstantOpt;
+
+      this.Name = problemData.Name + string.Format("lambda={0:N2}", lambda);
+
+      this.N = problemData.TrainingIndices.Count();
+      this.d = problemData.AllowedInputVariables.Count();
+      if (d > 26) throw new NotSupportedException(); // we only allow single-character terminal symbols so far
+      this.x = new double[N, d];
+      this.y = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).ToArray();
+
+      var varEst = new OnlineMeanAndVarianceCalculator();
+
+      var means = new double[d];
+      var stdDevs = new double[d];
+
+      int i = 0;
+      foreach (var inputVariable in problemData.AllowedInputVariables) {
+        varEst.Reset();
+        problemData.Dataset.GetDoubleValues(inputVariable).ToList().ForEach(varEst.Add);
+        if (varEst.VarianceErrorState != OnlineCalculatorError.None) throw new ArgumentException();
+        means[i] = varEst.Mean;
+        stdDevs[i] = Math.Sqrt(varEst.Variance);
+        i++;
+      }
+
+      i = 0;
+      foreach (var r in problemData.TrainingIndices) {
+        int j = 0;
+        foreach (var inputVariable in problemData.AllowedInputVariables) {
+          x[i, j] = (problemData.Dataset.GetDoubleValue(inputVariable, r) - means[j]) / stdDevs[j];
+          j++;
+        }
+        i++;
+      }
+
+      this.random = random;
+      this.lambda = lambda;
+
+      // initialize ERC values
+      erc = Enumerable.Range(0, 10).Select(_ => Rand.RandNormal(random) * 10).ToArray();
+
+      char firstVar = 'a';
+      char lastVar = Convert.ToChar(Convert.ToByte('a') + d - 1);
+      if (!useConstantOpt) {
+        this.grammar = new Grammar(grammarString.Replace("<variables>", firstVar + " .. " + lastVar));
+        this.TreeBasedGPGrammar = new Grammar(treeGrammarString.Replace("<variables>", firstVar + " .. " + lastVar));
+      } else {
+        this.grammar = new Grammar(grammarConstOptString.Replace("<variables>", firstVar + " .. " + lastVar));
+        this.TreeBasedGPGrammar = new Grammar(treeGrammarConstOptString.Replace("<variables>", firstVar + " .. " + lastVar));
+      }
+    }
+
+
+    public double BestKnownQuality(int maxLen) {
+      // for now only an upper bound is returned, ideally we have an R² of 1.0
+      return 1.0;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      var extender = new ExpressionExtender();
+      sentence = extender.CanonicalRepresentation(sentence);
+      if (useConstantOpt)
+        return OptimizeConstantsAndEvaluate(sentence);
+      else {
+        Debug.Assert(SimpleEvaluate(sentence) == SimpleEvaluate(extender.CanonicalRepresentation(sentence)));
+        return SimpleEvaluate(sentence);
+      }
+    }
+
+    public double SimpleEvaluate(string sentence) {
+      var interpreter = new ExpressionInterpreter();
+      var rowData = new double[d];
+      return HeuristicLab.Common.Extensions.RSq(y, Enumerable.Range(0, N).Select(i => {
+        for (int j = 0; j < d; j++) rowData[j] = x[i, j];
+        return interpreter.Interpret(sentence, rowData, erc);
+      }));
+    }
+
+
+    public string CanonicalRepresentation(string phrase) {
+      var extender = new ExpressionExtender();
+      return extender.CanonicalRepresentation(phrase);
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      throw new NotImplementedException();
+      //phrase = CanonicalRepresentation(phrase);
+      //return phrase.Split('+').Distinct().Select(t => new Feature(t, 1.0));
+      // return new Feature[] { new Feature(phrase, 1.0) };
+    }
+
+
+
+    public double OptimizeConstantsAndEvaluate(string sentence) {
+      AutoDiff.Term func;
+      int n = x.GetLength(0);
+      int m = x.GetLength(1);
+
+      var compiler = new ExpressionCompiler();
+      Variable[] variables = Enumerable.Range(0, m).Select(_ => new Variable()).ToArray();
+      Variable[] constants;
+      compiler.Compile(sentence, out func, variables, out constants);
+
+      // constants are manipulated 
+
+      if (!constants.Any()) return SimpleEvaluate(sentence);
+
+      // L2 regularization
+      // not possible with lsfit, would need to change to minlm below
+      // func = TermBuilder.Sum(func, lambda * TermBuilder.Sum(constants.Select(con => con * con)));
+
+      AutoDiff.IParametricCompiledTerm compiledFunc = func.Compile(constants, variables); // variate constants, leave variables fixed to data
+
+      // 10 restarts with random starting points
+      double[] bestStart = null;
+      double bestError = double.PositiveInfinity;
+      int info;
+      alglib.lsfitstate state;
+      alglib.lsfitreport rep;
+      alglib.ndimensional_pfunc function_cx_1_func = CreatePFunc(compiledFunc);
+      alglib.ndimensional_pgrad function_cx_1_grad = CreatePGrad(compiledFunc);
+      for (int t = 0; t < 10; t++) {
+        double[] cStart = constants.Select(_ => Rand.RandNormal(random) * 10).ToArray();
+        double[] cEnd;
+        // start with normally distributed (N(0, 10)) weights
+
+
+        int k = cStart.Length;
+
+
+        const int maxIterations = 10;
+        try {
+          alglib.lsfitcreatefg(x, y, cStart, n, m, k, false, out state);
+          alglib.lsfitsetcond(state, 0.0, 0.0, maxIterations);
+          //alglib.lsfitsetgradientcheck(state, 0.001);
+          alglib.lsfitfit(state, function_cx_1_func, function_cx_1_grad, null, null);
+          alglib.lsfitresults(state, out info, out cEnd, out rep);
+          if (info != -7 && rep.rmserror < bestError) {
+            bestStart = cStart;
+            bestError = rep.rmserror;
+          }
+        } catch (ArithmeticException) {
+          return 0.0;
+        } catch (alglib.alglibexception) {
+          return 0.0;
+        }
+      }
+
+      // 100 iteration steps from the best starting point
+      {
+        double[] c = bestStart;
+
+        int k = c.Length;
+
+        const int maxIterations = 100;
+        try {
+          alglib.lsfitcreatefg(x, y, c, n, m, k, false, out state);
+          alglib.lsfitsetcond(state, 0.0, 0.0, maxIterations);
+          //alglib.lsfitsetgradientcheck(state, 0.001);
+          alglib.lsfitfit(state, function_cx_1_func, function_cx_1_grad, null, null);
+          alglib.lsfitresults(state, out info, out c, out rep);
+        } catch (ArithmeticException) {
+          return 0.0;
+        } catch (alglib.alglibexception) {
+          return 0.0;
+        }
+        //info == -7  => constant optimization failed due to wrong gradient
+        if (info == -7) throw new ArgumentException();
+        {
+          var rowData = new double[d];
+          return HeuristicLab.Common.Extensions.RSq(y, Enumerable.Range(0, N).Select(i => {
+            for (int j = 0; j < d; j++) rowData[j] = x[i, j];
+            return compiledFunc.Evaluate(c, rowData);
+          }));
+        }
+      }
+    }
+
+
+    private static alglib.ndimensional_pfunc CreatePFunc(AutoDiff.IParametricCompiledTerm compiledFunc) {
+      return (double[] c, double[] x, ref double func, object o) => {
+        func = compiledFunc.Evaluate(c, x);
+      };
+    }
+
+    private static alglib.ndimensional_pgrad CreatePGrad(AutoDiff.IParametricCompiledTerm compiledFunc) {
+      return (double[] c, double[] x, ref double func, double[] grad, object o) => {
+        var tupel = compiledFunc.Differentiate(c, x);
+        func = tupel.Item2;
+        Array.Copy(tupel.Item1, grad, grad.Length);
+      };
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+
+      TreeToSentence(tree.Root.GetSubtree(0).GetSubtree(0), sb);
+
+      return sb.ToString();
+    }
+
+
+    private void TreeToSentence(ISymbolicExpressionTreeNode treeNode, StringBuilder sb) {
+      if (treeNode.SubtreeCount == 0) {
+        // terminal
+        sb.Append(treeNode.Symbol.Name);
+      } else {
+        string op = string.Empty;
+        switch (treeNode.Symbol.Name) {
+          case "S": op = "+"; break;
+          case "N": op = "-"; break;
+          case "P": op = "*"; break;
+          case "D": op = "%"; break;
+          default: {
+              Debug.Assert(treeNode.SubtreeCount == 1);
+              break;
+            }
+        }
+        // nonterminal
+        if (treeNode.SubtreeCount > 1) sb.Append("(");
+        TreeToSentence(treeNode.Subtrees.First(), sb);
+        foreach (var subTree in treeNode.Subtrees.Skip(1)) {
+          sb.Append(op);
+          TreeToSentence(subTree, sb);
+        }
+        if (treeNode.SubtreeCount > 1) sb.Append(")");
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/ExpressionInterpreter.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/ExpressionInterpreter.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/ExpressionInterpreter.cs	(revision 12290)
@@ -0,0 +1,143 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics.Eventing.Reader;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.Remoting.Messaging;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class ExpressionInterpreter {
+    private static readonly double[] emptyErc = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
+
+    private string sentence;
+    private int syIdx;
+    // interprets sentences from L(G(Expr)):
+    // Expr -> Term { ('+' | '-' | '^' ) Term } 
+    // Term -> Fact { ('*' | '%') Fact }
+    // Fact -> '!' Expr | '(' Expr ')' | Var | const
+    // Var -> 'a'..'z'
+    // const -> '0' .. '9'
+
+    // uses protected division symbol %
+    // constants are Koza-style ephemeral random constants (ERC). for now we only allow up to 10 constants.
+    // The constant symbols '0' .. '9' are treated as ERC indices
+
+    // for boolean problems the symbol * representes the AND operator, the symbol + represents the OR operator, ! = NOT, ^ = XOR
+
+    public bool Interpret(string sentence, bool[] vars) {
+      return Interpret(sentence, vars, emptyErc);
+    }
+
+    public bool Interpret(string sentence, bool[] vars, double[] erc) {
+      InitLex(sentence);
+      var d = new double[vars.Length];
+      for (int i = 0; i < vars.Length; i++) d[i] = vars[i] ? 1.0 : 0.0;
+      return !DoubleExtensions.IsAlmost(Expr(d, erc), 0);
+    }
+
+    public double Interpret(string sentence, double[] vars) {
+      return Interpret(sentence, vars, emptyErc);
+    }
+
+    public double Interpret(string sentence, double[] vars, double[] erc) {
+      InitLex(sentence);
+      return Expr(vars, erc);
+    }
+
+
+    private void InitLex(string sentence) {
+      this.sentence = sentence;
+      this.syIdx = 0;
+    }
+
+    private char CurSy() {
+      if (syIdx >= sentence.Length) return '\0';
+      return sentence[syIdx];
+    }
+    private void NewSy() {
+      if (syIdx < sentence.Length) syIdx++;
+    }
+
+    // helper for xor
+    private double Not(double x) {
+      return DoubleExtensions.IsAlmost(x, 0) ? 1.0 : 0.0;
+    }
+
+    private double Expr(double[] d, double[] erc) {
+      var r = 0.0;
+      r = Term(d, erc);
+      var curSy = CurSy();
+      while (curSy == '+' || curSy == '-' || curSy == '^') {
+        if (curSy == '+') {
+          NewSy();
+          r += Term(d, erc);
+        } else if (curSy == '-') {
+          NewSy();
+          r -= Term(d, erc);
+        } else {
+          NewSy();
+          var e = Expr(d, erc);
+          r = Not(r) * e + r * Not(e); // xor = (!x AND y) OR (x AND !y)
+        }
+        curSy = CurSy();
+      }
+      return r;
+    }
+
+    private double Term(double[] d, double[] erc) {
+      var r = 0.0;
+      r = Fact(d, erc);
+      var curSy = CurSy();
+      while (curSy == '*' || curSy == '%') {
+        if (curSy == '*') {
+          NewSy();
+          r *= Fact(d, erc);
+        } else {
+          NewSy();
+          var nom = Fact(d, erc);
+          if (HeuristicLab.Common.Extensions.IsAlmost(nom, 0.0)) nom = 1.0;
+          r /= nom;
+        }
+        curSy = CurSy();
+      }
+      return r;
+    }
+
+    private double Fact(double[] d, double[] erc) {
+      double r = 0.0;
+      var curSy = CurSy();
+      if (curSy == '!') {
+        NewSy();
+        r = Not(Expr(d, erc));
+      } else if (curSy == '(') {
+        NewSy();
+        r = Expr(d, erc);
+        if (CurSy() != ')') throw new ArgumentException();
+        NewSy();
+      } else if (curSy >= 'a' && curSy <= 'z') {
+        int o = (byte)curSy - (byte)'a';
+        //int o = Convert.ToByte(CurSy()) - Convert.ToByte('a');
+        if (o < 0 || o >= d.Length) throw new ArgumentException();
+        r = d[o];
+        NewSy();
+      } else if (curSy == '/') {
+        // /-symbol is used in the expressionextender to represent inverse (1/x). 
+        // this is necessary because we also use symbols 0..9 as indices for ERCs
+        NewSy();
+        r = 1.0 / Fact(d, erc);
+      } else /* if (curSy >= '0' && curSy <= '9') */ {
+        int o = (byte)curSy - (byte)'0';
+        //int o = Convert.ToByte(CurSy()) - Convert.ToByte('a');
+        if (o < 0 || o >= 10) throw new ArgumentException();
+        r = erc[o];
+        NewSy();
+      }
+      //} else throw new ArgumentException();
+      return r;
+    }
+
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Feature.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Feature.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Feature.cs	(revision 12290)
@@ -0,0 +1,22 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // for the feature extraction function for grammatical optimization problems (sparse representation)
+  public class Feature {
+    public string Id { get; private set; }
+    public double Value { get; private set; }
+
+    public Feature(string id, double val) {
+      this.Id = id;
+      this.Value = val;
+    }
+
+    public override string ToString() {
+      return string.Format("{0} {1:N3}", Id, Value);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Grammar.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Grammar.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Grammar.cs	(revision 12290)
@@ -0,0 +1,287 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+
+  // phrases and symbols are both strings
+  // spaces are used to separate symbols in phases
+  // terminal and non-terminal symbols must not contain white-space
+  // white-spaces are not counted when determining the length of a sentence, only the number of terminal symbols is relevant
+
+  public class Grammar : IGrammar {
+
+    private readonly Dictionary<char, List<Sequence>> rules;
+    private readonly HashSet<char> terminalSymbols;
+    private readonly char sentenceSymbol;
+    private readonly HashSet<char> nonTerminalSymbols;
+    private readonly Dictionary<Sequence, int> maxPhraseLength = new Dictionary<Sequence, int>();
+    private readonly Dictionary<Sequence, int> minPhraseLength = new Dictionary<Sequence, int>();
+
+    public char SentenceSymbol { get { return sentenceSymbol; } }
+    public IEnumerable<char> NonTerminalSymbols { get { return nonTerminalSymbols; } }
+    public IEnumerable<char> TerminalSymbols { get { return terminalSymbols; } }
+    public IEnumerable<char> Symbols { get { return nonTerminalSymbols.Concat(terminalSymbols); } } // ctor guarantees that intersection is empty
+
+    // cloning ctor
+    public Grammar(Grammar orig) {
+      this.rules = new Dictionary<char, List<Sequence>>();
+      foreach (var r in orig.rules)
+        this.rules.Add(r.Key, new List<Sequence>(r.Value.Select(v => new ReadonlySequence(v)))); // clone sequences
+      this.terminalSymbols = new HashSet<char>(orig.terminalSymbols);
+      this.sentenceSymbol = orig.sentenceSymbol;
+      this.nonTerminalSymbols = new HashSet<char>(orig.nonTerminalSymbols);
+      this.maxPhraseLength = new Dictionary<Sequence, int>();
+      foreach (var p in orig.maxPhraseLength) this.maxPhraseLength.Add(new ReadonlySequence(p.Key), p.Value);
+      this.minPhraseLength = new Dictionary<Sequence, int>();
+      foreach (var p in orig.minPhraseLength) this.minPhraseLength.Add(new ReadonlySequence(p.Key), p.Value);
+    }
+
+    public Grammar(string grammar) : this((Grammar)FromString(grammar)) { }
+
+    public Grammar(char sentenceSymbol, IEnumerable<char> terminalSymbols, IEnumerable<char> nonTerminalSymbols, IEnumerable<Tuple<char, string>> rules) {
+      this.sentenceSymbol = sentenceSymbol;
+      this.terminalSymbols = new HashSet<char>(terminalSymbols);
+      this.nonTerminalSymbols = new HashSet<char>(nonTerminalSymbols);
+      this.rules = new Dictionary<char, List<Sequence>>();
+      foreach (var r in rules) {
+        if (!this.rules.ContainsKey(r.Item1)) this.rules.Add(r.Item1, new List<Sequence>());
+        this.rules[r.Item1].Add(new ReadonlySequence(r.Item2)); // here we store an array of symbols for a phase
+      }
+
+      CheckValidity();
+      CalculatePhraseLengthBounds();
+    }
+
+    private void CheckValidity() {
+      // check validity of parameters
+      if (!nonTerminalSymbols.Contains(sentenceSymbol) || // sentence symbol must be a non-terminal
+          nonTerminalSymbols.Any(c => c < 'A' || c > 'Z') || // nonterminal symbols \in 'A' .. 'Z'
+          terminalSymbols.Intersect(nonTerminalSymbols).Any() || // no overlap between non-terminals and terminal
+          rules.Keys.Intersect(nonTerminalSymbols).Count() < nonTerminalSymbols.Count ||
+          nonTerminalSymbols.Any(nt => !rules.ContainsKey(nt)) || // each nt must have at least one rule
+          rules.Values.Any(alternatives => !alternatives.Any()) || // no eps alternatives
+          rules.Any(rule => rule.Value.Any(alternative => alternative.Length == 1 && alternative[0] == rule.Key))
+        // no infinite direct recursions
+        ) {
+        throw new ArgumentException();
+      }
+    }
+
+    private void CalculatePhraseLengthBounds() {
+      // cache phrase lengths
+      foreach (var nt in nonTerminalSymbols) {
+        var min = int.MaxValue;
+        var max = int.MinValue;
+        foreach (var alt in rules[nt].OrderBy(alt => alt.Length)) {
+          CalcAndSetMinPhraseLength(alt);
+          CalcAndSetMaxPhraseLength(alt);
+
+          min = Math.Min(min, minPhraseLength[alt]);
+          max = Math.Max(max, maxPhraseLength[alt]);
+        }
+        minPhraseLength[new ReadonlySequence(nt)] = min;
+        maxPhraseLength[new ReadonlySequence(nt)] = max;
+      }
+    }
+
+    public IEnumerable<Sequence> GetAlternatives(char nt) {
+      return rules[nt];
+    }
+
+    public IEnumerable<Sequence> GetTerminalAlternatives(char nt) {
+      return GetAlternatives(nt).Where(alt => alt.All(IsTerminal));
+    }
+
+    public IEnumerable<Sequence> GetNonTerminalAlternatives(char nt) {
+      return GetAlternatives(nt).Where(alt => alt.Any(IsNonTerminal));
+    }
+
+    #region population of minphraselength cache
+    private int CalcAndSetMinSymbolLength(char symb) {
+      if (IsTerminal(symb)) return 1;
+      else return Math.Min(short.MaxValue, rules[symb].Min(alt => CalcAndSetMinPhraseLength(alt))); // maximal allowed value is short.MaxValue
+    }
+    private int CalcAndSetMinPhraseLength(Sequence phrase) {
+      Debug.Assert(phrase is ReadonlySequence);
+      int l;
+      if (minPhraseLength.TryGetValue(phrase, out l)) return l;
+      l = 0;
+      minPhraseLength[phrase] = short.MaxValue; // mark
+
+      foreach (var symb in phrase) {
+        l += CalcAndSetMinSymbolLength(symb);
+      }
+
+      l = Math.Min(short.MaxValue, l); // maximal allowed value is short.MaxValue
+      minPhraseLength[phrase] = l; // update 
+      return l;
+    }
+    #endregion
+
+    // read only access to caches
+    private int MinSymbolLength(char symb) {
+      if (IsTerminal(symb)) return 1;
+      else return Math.Min(short.MaxValue, rules[symb].Min(alt => MinPhraseLength(alt))); // maximal allowed value is short.MaxValue
+    }
+    public int MinPhraseLength(Sequence phrase) {
+      var minLen = 0;
+      if (minPhraseLength.TryGetValue(phrase, out minLen)) return minLen;
+      foreach (var s in phrase) {
+        minLen += MinSymbolLength(s);
+      }
+      return Math.Min(short.MaxValue, minLen); // maximal allowed value is short.MaxValue
+    }
+
+    #region population of maxphraselength cache
+    private int CalcAndSetMaxSymbolLength(char symb) {
+      if (IsTerminal(symb)) return 1;
+      else return Math.Min(short.MaxValue, rules[symb].Max(alt => CalcAndSetMaxPhraseLength(alt))); // maximal allowed value is short.MaxValue
+    }
+    // caches for this are build in construction of object
+    private int CalcAndSetMaxPhraseLength(Sequence phrase) {
+      Debug.Assert(phrase is ReadonlySequence);
+      int l;
+      if (maxPhraseLength.TryGetValue(phrase, out l)) return l;
+      l = 0;
+      maxPhraseLength[phrase] = short.MaxValue; // mark
+
+      foreach (var symb in phrase) {
+        l += CalcAndSetMaxSymbolLength(symb);
+      }
+      l = Math.Min(short.MaxValue, l); // maximal allowed value is short.MaxValue
+      maxPhraseLength[phrase] = l; // update 
+      return l;
+    }
+    #endregion
+
+    // read only access to caches
+    public int MaxSymbolLength(char symb) {
+      if (IsTerminal(symb)) return 1;
+      else return Math.Min(short.MaxValue, rules[symb].Max(alt => MaxPhraseLength(alt))); // maximal allowed value is short.MaxValue 
+    }
+    public int MaxPhraseLength(Sequence phrase) {
+      var maxLen = 0;
+      if (maxPhraseLength.TryGetValue(phrase, out maxLen)) return maxLen;
+      foreach (var s in phrase) {
+        maxLen += MaxSymbolLength(s);
+      }
+      return Math.Min(short.MaxValue, maxLen); // maximal allowed value is short.MaxValue
+    }
+
+    public Sequence CompleteSentenceRandomly(Random random, Sequence phrase, int maxLen) {
+      if (phrase.Length > maxLen) throw new ArgumentException();
+      if (MinPhraseLength(phrase) > maxLen) throw new ArgumentException();
+      while (!phrase.IsTerminal) {
+        char nt = phrase.FirstNonTerminal;
+
+        int maxLenOfReplacement = maxLen - (phrase.Length - 1); // replacing aAb with maxLen 4 means we can only use alternatives with a minPhraseLen <= 2
+        Debug.Assert(maxLenOfReplacement > 0);
+
+        var alts = GetAlternatives(nt).Where(alt => MinPhraseLength(alt) <= maxLenOfReplacement);
+        Debug.Assert(alts.Any());
+
+        // replace nt with random alternative
+        var selectedAlt = alts.SelectRandom(random);
+        phrase.ReplaceAt(phrase.FirstNonTerminalIndex, 1, selectedAlt);
+
+      }
+      return phrase;
+    }
+
+    public bool IsTerminal(char symbol) {
+      return terminalSymbols.Contains(symbol);
+    }
+
+    public bool IsTerminal(string phrase) {
+      // reverse because for our grammars and left-canonical derivation it is more likely that NTs occur near the end of the sequence
+      for (int i = phrase.Length - 1; i >= 0; i--) {
+        if (!IsTerminal(phrase[i])) return false;
+      }
+      return true;
+    }
+
+    public bool IsNonTerminal(char symbol) {
+      return nonTerminalSymbols.Contains(symbol);
+    }
+
+    private static readonly string symbRegex = @"(?<symb>[^\s\|\.])";
+    private static readonly string altRegex = @"(?<alt>[^\s\|\.]+)";
+    private static readonly string altListRegex = altRegex + @"(\s*\|\s*" + altRegex + @")*";
+    // private static readonly string terminalClassRegex = @"[^\s\|\.]\s*\.\.\s*[^\s\|\.]";
+    private static readonly string terminalClassRegex = symbRegex + @"\s*\.\.\s*" + symbRegex + @"\s*";
+
+    private static readonly Regex sentenceSymbRegex = new Regex(@"\s*G\(" + symbRegex + @"\):\s*$", RegexOptions.ExplicitCapture);
+    private static readonly Regex ruleRegex = new Regex(@"\s*" + symbRegex + @"\s*->\s*(" + altListRegex + "|" + terminalClassRegex + @")\s*$", RegexOptions.ExplicitCapture);
+
+
+    // supports only meta-symbol for alternatives (syntax of formal grammars)
+    // epsilon-alternatives are not supported
+    public static IGrammar FromString(string grammar) {
+      char sentenceSymbol;
+      var nonTerminals = new List<char>();
+      var terminals = new List<char>();
+      var rules = new List<Tuple<char, string>>();
+
+      using (var reader = new StringReader(grammar)) {
+        var line = reader.ReadLine();
+        while (string.IsNullOrWhiteSpace(line)) line = reader.ReadLine();
+        // first line contains "G(...):"
+        var match = sentenceSymbRegex.Match(line);
+        if (!match.Success) throw new ArgumentException();
+        sentenceSymbol = match.Groups["symb"].Value[0]; // only a single symbol is in the capture group
+
+        line = reader.ReadLine();
+        while (!string.IsNullOrWhiteSpace(line)) {
+          match = ruleRegex.Match(line);
+          if (match.Length != line.Length) throw new ArgumentException();
+          var nt = match.Groups["symb"].Captures[0].Value[0];
+          if (!nonTerminals.Contains(nt)) nonTerminals.Add(nt);
+
+          // contains alternatives?
+          if (match.Groups["alt"].Captures.Count > 0) {
+            foreach (var m in match.Groups["alt"].Captures) {
+              rules.Add(Tuple.Create(nt, m.ToString()));
+            }
+          } else if (match.Groups["symb"].Captures.Count == 3) {
+            // contains terminal range
+            var fromSymb = match.Groups["symb"].Captures[1].Value[0];
+            var toSymb = match.Groups["symb"].Captures[2].Value[0];
+            if (fromSymb > toSymb) throw new ArgumentException("left symbol is larger than right symbol in range");
+
+            for (byte b = Convert.ToByte(fromSymb); b <= Convert.ToByte(toSymb); b++) {
+              rules.Add(Tuple.Create(nt, Convert.ToChar(b).ToString()));
+            }
+          }
+
+          line = reader.ReadLine();
+        }
+      }
+
+      foreach (var r in rules) {
+        foreach (var s in r.Item2) {
+          if (!nonTerminals.Contains(s) && !terminals.Contains(s)) terminals.Add(s);
+        }
+      }
+      return new Grammar(sentenceSymbol, terminals, nonTerminals, rules);
+    }
+
+    public override string ToString() {
+      // for debugging
+      var sb = new StringBuilder();
+      sb.AppendFormat("G({0}):", sentenceSymbol).AppendLine();
+      foreach (var r in rules) {
+        foreach (var alt in r.Value) {
+          sb.AppendFormat("  {0} -> {1} (min: {2}, max {3})", r.Key, alt, MinPhraseLength(alt), MaxPhraseLength(alt))
+            .AppendLine();
+        }
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/HeuristicLab.Problems.GrammaticalOptimization.csproj	(revision 12290)
@@ -0,0 +1,93 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{CB9DCCF6-667E-4A13-B82D-DBD6B45A045E}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Problems.GrammaticalOptimization</RootNamespace>
+    <AssemblyName>HeuristicLab.Problems.GrammaticalOptimization</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="HeuristicLab.Common-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Common-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Core-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Core-3.3.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4, Version=3.4.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\HeuristicLab\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="PartialExpressionInterpreter.cs" />
+    <Compile Include="ExpressionInterpreter.cs" />
+    <Compile Include="Feature.cs" />
+    <Compile Include="Interfaces\IGrammar.cs" />
+    <Compile Include="Interfaces\IProblem.cs" />
+    <Compile Include="Interfaces\ISymbolicExpressionTreeProblem.cs" />
+    <Compile Include="Problems\PrimePolynomialProblem.cs" />
+    <Compile Include="Problems\PermutationProblem.cs" />
+    <Compile Include="Problems\EvenParityProblem.cs" />
+    <Compile Include="Problems\FindPhrasesProblem.cs" />
+    <Compile Include="Problems\HardPalindromeProblem.cs" />
+    <Compile Include="Problems\PalindromeProblem.cs" />
+    <Compile Include="Problems\RoyalPairProblem.cs" />
+    <Compile Include="Problems\RoyalPhraseSequenceProblem.cs" />
+    <Compile Include="Problems\RoyalRoadProblem.cs" />
+    <Compile Include="Problems\RoyalSequenceProblem.cs" />
+    <Compile Include="Problems\RoyalSymbolProblem.cs" />
+    <Compile Include="Problems\RoyalTreeProblem.cs" />
+    <Compile Include="Problems\SantaFeAntProblem.cs" />
+    <Compile Include="Problems\SymbolicRegressionPoly10Problem.cs" />
+    <Compile Include="Grammar.cs" />
+    <Compile Include="ReadonlySequence.cs" />
+    <Compile Include="SentenceSetStatistics.cs" />
+    <Compile Include="Sequence.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup />
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IGrammar.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IGrammar.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IGrammar.cs	(revision 12290)
@@ -0,0 +1,28 @@
+﻿using System;
+using System.Collections.Generic;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+
+  // symbols are characters, possible nonterminal symbols are : 'A'..'Z'
+  // phrases and sentences are strings
+  // each symbol is only on element in a string
+
+  public interface IGrammar {
+    char SentenceSymbol { get; }
+    IEnumerable<char> NonTerminalSymbols { get; }
+    IEnumerable<char> TerminalSymbols { get; }
+    IEnumerable<char> Symbols { get; }
+
+    IEnumerable<Sequence> GetAlternatives(char nt);
+    IEnumerable<Sequence> GetTerminalAlternatives(char nt);
+    IEnumerable<Sequence> GetNonTerminalAlternatives(char nt);
+
+    int MinPhraseLength(Sequence phrase);
+    int MaxPhraseLength(Sequence phrase);
+    Sequence CompleteSentenceRandomly(Random random, Sequence phrase, int maxLen);
+
+    bool IsTerminal(char symbol);
+    bool IsTerminal(string phrase);
+    bool IsNonTerminal(char symbol);
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/IProblem.cs	(revision 12290)
@@ -0,0 +1,17 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // represents a grammatical optimization problem
+  public interface IProblem {
+    string Name { get; }
+    double BestKnownQuality(int maxLen);
+    IGrammar Grammar { get; }
+    double Evaluate(string sentence);
+    string CanonicalRepresentation(string phrase); // canonical state must use correct syntax (must be a valid input for evaluate)
+    IEnumerable<Feature> GetFeatures(string phrase);
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/ISymbolicExpressionTreeProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/ISymbolicExpressionTreeProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Interfaces/ISymbolicExpressionTreeProblem.cs	(revision 12290)
@@ -0,0 +1,13 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // represents a grammatical optimization problem that can also be used with tree-based GP in HeuristicLab
+  public interface ISymbolicExpressionTreeProblem : IProblem {
+    IGrammar TreeBasedGPGrammar { get; } // grammar for HL GP
+    string ConvertTreeToSentence(ISymbolicExpressionTree tree);
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/EvenParityProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/EvenParityProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/EvenParityProblem.cs	(revision 12290)
@@ -0,0 +1,131 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // 4-bit even parity
+  public class EvenParityProblem : ISymbolicExpressionTreeProblem {
+    // + == OR
+    // * == AND
+    private const string grammarString = @"
+G(S):
+S -> a | b | c | d | a*S | b*S | c*S | d*S | a+S | b+S | c+S | d+S | !S | (S)
+";
+    // A = AND, O = OR, N = NOT, C = Clause
+    private const string hlGrammarString = @"
+G(E):
+E -> A | O | N | C | a | b | c | d 
+A -> EE | EEE
+O -> EE | EEE
+N -> E
+C -> E
+    ";
+
+    private readonly IGrammar grammar;
+    public string Name { get { return "EvenParity"; } }
+
+    public EvenParityProblem() {
+      this.grammar = new Grammar(grammarString);
+      this.TreeBasedGPGrammar = new Grammar(hlGrammarString);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      // for now only an upper bound is returned, ideally all fitness cases are predicted correctly
+      return 16;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      var interpreter = new ExpressionInterpreter(); // for concurrent evaluation
+      var vars = new bool[4];
+      var nCorrect = 0;
+      for (int b0 = 0; b0 <= 1; b0++)
+        for (int b1 = 0; b1 <= 1; b1++)
+          for (int b2 = 0; b2 <= 1; b2++)
+            for (int b3 = 0; b3 <= 1; b3++) {
+              vars[0] = b0 > 0;
+              vars[1] = b1 > 0;
+              vars[2] = b2 > 0;
+              vars[3] = b3 > 0;
+
+              var pred = interpreter.Interpret(sentence, vars);
+              var target = (b0 > 0) ^ (b1 > 0) ^ (b2 > 0) ^ (b3 > 0);
+              if (pred == target) nCorrect++;
+            }
+      return nCorrect;
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      throw new NotImplementedException();
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase)
+    {
+      return new[] {new Feature(phrase, 1.0)};
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+
+      TreeToSentence(tree.Root.GetSubtree(0).GetSubtree(0), sb);
+
+      return sb.ToString();
+    }
+
+    private void TreeToSentence(ISymbolicExpressionTreeNode treeNode, StringBuilder sb) {
+      if (treeNode.SubtreeCount == 0) {
+        // terminal
+        sb.Append(treeNode.Symbol.Name);
+      } else {
+        switch (treeNode.Symbol.Name) {
+          case "O": {
+              sb.Append("(");
+              TreeToSentence(treeNode.Subtrees.First(), sb);
+              foreach (var subTree in treeNode.Subtrees.Skip(1)) {
+                sb.Append("+");
+                TreeToSentence(subTree, sb);
+              }
+              sb.Append(")");
+              break;
+            }
+          case "A": {
+              TreeToSentence(treeNode.Subtrees.First(), sb);
+              foreach (var subTree in treeNode.Subtrees.Skip(1)) {
+                sb.Append("*");
+                TreeToSentence(subTree, sb);
+              }
+              break;
+            }
+          case "N": {
+              Debug.Assert(treeNode.SubtreeCount == 1);
+              sb.Append("!(");
+              TreeToSentence(treeNode.Subtrees.Single(), sb);
+              sb.Append(")");
+              break;
+            }
+          case "C": {
+              Debug.Assert(treeNode.SubtreeCount == 1);
+              sb.Append("(");
+              TreeToSentence(treeNode.Subtrees.Single(), sb);
+              sb.Append(")");
+              break;
+            }
+          default: {
+              Debug.Assert(treeNode.SubtreeCount == 1);
+              TreeToSentence(treeNode.Subtrees.Single(), sb);
+              break;
+            }
+        }
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/FindPhrasesProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/FindPhrasesProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/FindPhrasesProblem.cs	(revision 12290)
@@ -0,0 +1,181 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // must find a set of phrases where the ordering of phrases is irrelevant
+  // Parameters
+  // - size of the alphabet
+  // - phrase length
+  // - number of phrases in the sequence
+  // - number of optimal phrases
+  // - reward for optimal phrases
+  // - number of decoy (sub-optimal) phrases
+  // - reward for decoy phrases (must be smaller than reward for optimal phrases)
+  // - phrasesAsSets: a switch to determine wether symbols in a phrase can be shuffled (sets) or if the ordering is relevant (non-sets)
+
+  // this problem should be similar to symbolic regression and should be easier for approaches using a state esimation value and the canoncial state 
+  // when phrases are symbol sets instead of sequences then value-estimation routines should be better (TD)
+  public class FindPhrasesProblem : ISymbolicExpressionTreeProblem {
+
+    private readonly IGrammar grammar;
+    private readonly int numPhrases;
+    private readonly int phraseLen;
+    private readonly double correctReward;
+    private readonly double decoyReward;
+    private readonly bool phrasesAsSets;
+    private readonly SortedSet<string> optimalPhrases;
+    private readonly SortedSet<string> decoyPhrases;
+    public string Name { get { return "FindPhrases"; } }
+
+    public FindPhrasesProblem(Random rand, int alphabetSize, int numPhrases, int phraseLen, int numOptimalPhrases, int numDecoyPhrases = 1,
+      double correctReward = 1.0, double decoyReward = 0.0, bool phrasesAsSets = false) {
+      if (alphabetSize <= 0 || alphabetSize > 26) throw new ArgumentException();
+      if (numPhrases <= 0) throw new ArgumentException();
+      if (phraseLen < 1) throw new ArgumentException();
+      if (numOptimalPhrases < numPhrases) throw new ArgumentException();
+      if (numDecoyPhrases < 0) throw new ArgumentException();
+      if (correctReward <= decoyReward) throw new ArgumentException();
+
+      this.numPhrases = numPhrases;
+      this.phraseLen = phraseLen;
+      this.correctReward = correctReward;
+      this.decoyReward = decoyReward;
+      this.phrasesAsSets = phrasesAsSets;
+
+      var sentenceSymbol = 'S';
+      var terminalSymbols = Enumerable.Range(0, alphabetSize).Select(off => (char)((byte)'a' + off)).ToArray();
+      var nonTerminalSymbols = new char[] { sentenceSymbol };
+
+      {
+        // create grammar
+        // S -> a..z | aS .. zS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t + sentenceSymbol.ToString())));
+
+        this.grammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+      {
+        // create grammar for tree-based GP
+        // S -> a..z | SS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(new Tuple<char, string>[] { Tuple.Create(sentenceSymbol, sentenceSymbol.ToString() + sentenceSymbol) });
+
+        this.TreeBasedGPGrammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+
+      // generate optimal phrases
+      optimalPhrases = new SortedSet<string>();
+      while (optimalPhrases.Count < numOptimalPhrases) {
+        string phrase = "";
+        for (int l = 0; l < phraseLen; l++) {
+          phrase += terminalSymbols.SelectRandom(rand);
+        }
+        phrase = CanonicalPhrase(phrase);
+
+        // don't allow dups
+        if (!optimalPhrases.Contains(phrase)) optimalPhrases.Add(phrase);
+      }
+
+      // generate decoy phrases
+      decoyPhrases = new SortedSet<string>();
+      while (decoyPhrases.Count < numDecoyPhrases) {
+        string phrase = "";
+        for (int l = 0; l < phraseLen; l++) {
+          phrase += terminalSymbols.SelectRandom(rand);
+        }
+        phrase = CanonicalPhrase(phrase);
+
+        // don't allow dups
+        if (!optimalPhrases.Contains(phrase) && !decoyPhrases.Contains(phrase)) decoyPhrases.Add(phrase);
+      }
+
+      Debug.Assert(Evaluate(BestKnownSolution) / BestKnownQuality(phraseLen * numPhrases) == 1.0);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      return Math.Min(maxLen / phraseLen, numPhrases) * correctReward; // integer division
+    }
+
+    public string BestKnownSolution {
+      get { return string.Join("", optimalPhrases.Take(numPhrases)); }
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      // sentence must contain only terminal symbols, we are not checking if the sentence is syntactically valid here because it would be too slow!
+      Debug.Assert(sentence.Any(c => grammar.IsTerminal(c)));
+
+
+      // split the sentence in phrases
+      // phrases must not overlap in the sentence, multiple occurences of a phrase are not counted
+      // the order of phrases is not relevant
+      var numPhrases = sentence.Length / phraseLen;
+      var phrases = new SortedSet<string>();
+      for (int phraseIdx = 0; phraseIdx < numPhrases; phraseIdx++) {
+        var sentenceIdx = phraseIdx * phraseLen;
+        var phrase = sentence.Substring(sentenceIdx, phraseLen);
+        phrase = CanonicalPhrase(phrase);
+        if (!phrases.Contains(phrase)) phrases.Add(phrase);
+      }
+
+      // add reward for each correct phrase that occurs in the sentence 
+      // add reward for each decoy phrase that occurs in the sentence
+      var reward = phrases.Intersect(optimalPhrases).Count() * correctReward
+               + phrases.Intersect(decoyPhrases).Count() * decoyReward;
+
+      return reward;
+    }
+
+    // TODO: cache canonical phrases in most-recently used dictionary for increased performance (see symbolicregressionpoly10problem)
+    private string CanonicalPhrase(string phrase) {
+      if (phrasesAsSets) return string.Join("", phrase.OrderBy(ch => (byte)ch));
+      else return phrase;
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      // as the ordering of phrases does not matter we can reorder the phrases
+      // and remove duplicates
+      var numPhrases = phrase.Length / phraseLen;
+      var phrases = new SortedSet<string>();
+      for (int phraseIdx = 0; phraseIdx < numPhrases; phraseIdx++) {
+        var sentenceIdx = phraseIdx * phraseLen;
+        var subphrase = phrase.Substring(sentenceIdx, phraseLen);
+        subphrase = CanonicalPhrase(subphrase);
+        if (!phrases.Contains(subphrase)) phrases.Add(subphrase);
+      }
+      var remainder = phrase.Substring(numPhrases * phraseLen, phrase.Length - (numPhrases * phraseLen));
+      remainder = CanonicalPhrase(remainder);
+      if (!phrases.Contains(remainder)) phrases.Add(remainder);
+
+      return string.Join("", phrases);
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase)
+    {
+      return new Feature[] {new Feature(phrase, 1.0),};
+    }
+
+    public override string ToString() {
+      return string.Format("\"FindPhrasesProblem {0} {1} {2} {3:F2} {4} {5:F2} {6}\"", numPhrases, phraseLen,
+        optimalPhrases.Count, correctReward, decoyPhrases.Count, decoyReward, phrasesAsSets);
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/HardPalindromeProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/HardPalindromeProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/HardPalindromeProblem.cs	(revision 12290)
@@ -0,0 +1,68 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class HardPalindromeProblem : ISymbolicExpressionTreeProblem {
+    // length of the longest palindrome in the sentence + number of different symbols occurring in the palindrome
+    private const string grammarString = @"
+G(S):
+S -> T | TS
+T -> a .. z
+";
+    private const string hlGrammarString = @"
+G(S):
+S -> a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z | SS
+";
+
+    private readonly IGrammar grammar;
+    public string Name { get { return "HardPalindrome"; } }
+
+    public HardPalindromeProblem() {
+      this.grammar = new Grammar(grammarString);
+      this.TreeBasedGPGrammar = new Grammar(hlGrammarString);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      // the whole sentence is a palindrome + each symbol occurs only once or twice
+      // for odd total length the number of different characters can be larger than len/2 (aba)
+      // the second part is limited to 26 different characters
+      return maxLen + Math.Min(26, (maxLen + 1) / 2);
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      var palindrome = PalindromeProblem.LongestPalindromicSubstring(sentence);
+      var palindromeBytes = ASCIIEncoding.ASCII.GetBytes(palindrome);
+      var chrCount = new int[256];
+      foreach (var b in palindromeBytes) {
+        chrCount[b]++;
+      }
+      return palindrome.Length + chrCount.Count(c => c > 0);
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase)
+    {
+      throw new NotImplementedException();
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/PalindromeProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/PalindromeProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/PalindromeProblem.cs	(revision 12290)
@@ -0,0 +1,109 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class PalindromeProblem : ISymbolicExpressionTreeProblem {
+    // length of the longest palindrome in the sentence
+    private const string grammarString = @"
+G(S):
+S -> T | TS
+T -> a .. z
+";
+
+
+    private const string hlGrammarString = @"
+G(S):
+S -> a | b | c | d | e | f | g | h | i | j | k | l | m | n | o | p | q | r | s | t | u | v | w | x | y | z | SS
+";
+
+    private readonly IGrammar grammar;
+    public string Name { get { return "Palindrome"; } }
+    public PalindromeProblem() {
+      this.grammar = new Grammar(grammarString);
+      this.TreeBasedGPGrammar = new Grammar(hlGrammarString);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      // the whole sentence is a palindrome
+      return maxLen;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      return LongestPalindromicSubstring(sentence).Length;
+    }
+
+
+    private static string Preprocess(string s) {
+      var t = new char[2 * s.Length + 3];
+      t[0] = '$';
+      t[t.Length - 2] = '#';
+      t[t.Length - 1] = '@';
+      for (int i = 0; i < s.Length; i++) {
+        t[2 * (i) + 1] = '#';
+        t[2 * i + 2] = s[i];
+      }
+
+      return new string(t);
+    }
+
+    public static string LongestPalindromicSubstring(string str) {
+      var tLen = 2 * str.Length + 3;
+      var p = new int[tLen];
+      var t = Preprocess(str);
+
+      var center = 0;
+      var right = 0;
+      for (int i = 1; i < t.Length - 1; i++) {
+        var mirror = 2 * center - i;
+        if (right > i) p[i] = Math.Min(right - i, p[mirror]);
+        while (t[i + (1 + p[i])] == t[i - (1 + p[i])]) {
+          p[i]++;
+        }
+        if (i + p[i] > right) {
+          center = i;
+          right = i + p[i];
+        }
+      }
+      var len = 0;
+      center = 0;
+      for (int i = 0; i < t.Length - 1; i++) {
+        if (p[i] > len) {
+          center = i;
+          len = p[i];
+        }
+      }
+
+      var start = (center - 1 - len) / 2;
+      var result = new StringBuilder();
+      for (int i = 0; i < len; i++) {
+        result.Append(str[start + i]);
+      }
+      return result.ToString();
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      throw new NotImplementedException();
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/PermutationProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/PermutationProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/PermutationProblem.cs	(revision 12290)
@@ -0,0 +1,59 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class PermutationProblem : ISymbolicExpressionTreeProblem {
+    private const string grammarString = @"
+G(S):
+S -> 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f | 0S | 1S | 2S | 3S | 4S | 5S | 6S | 7S | 8S | 9S | aS | bS | cS | dS | eS | fS
+";
+
+    private const string hlGrammarString = @"
+G(S):
+S -> 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | a | b | c | d | e | f | SS
+";
+
+    private readonly IGrammar grammar;
+    public string Name { get { return "Permutation"; } }
+    public PermutationProblem() {
+      this.grammar = new Grammar(grammarString);
+      this.TreeBasedGPGrammar = new Grammar(hlGrammarString);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      return Math.Min(16.0, maxLen);
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      // sentence must contain only terminal symbols, we are not checking if the sentence is syntactically valid here because it would be too slow!
+      Debug.Assert(sentence.Any(c => grammar.IsTerminal(c)));
+      return sentence.Distinct().Count();
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      throw new NotImplementedException();
+    }
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPairProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPairProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPairProblem.cs	(revision 12290)
@@ -0,0 +1,85 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // counts the number of times a pair of symbols occurs in a sentence
+  public class RoyalPairProblem : ISymbolicExpressionTreeProblem {
+
+    private readonly IGrammar grammar;
+    private readonly int numTerminals;
+    public string Name { get { return "RoyalPair"; } }
+
+    public RoyalPairProblem(int numTerminals = 2) {
+      this.numTerminals = numTerminals;
+
+      var sentenceSymbol = 'S';
+      var terminalSymbols = Enumerable.Range(0, numTerminals).Select(off => (char)((byte)'a' + off)).ToArray();
+      var nonTerminalSymbols = new char[] { sentenceSymbol };
+
+      {
+        // create grammar
+        // S -> a..z | aS .. zS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t + sentenceSymbol.ToString())));
+
+        this.grammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+      {
+        // create grammar for tree-based GP
+        // S -> a..z | SS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(new Tuple<char, string>[] { Tuple.Create(sentenceSymbol, sentenceSymbol.ToString() + sentenceSymbol) });
+
+        this.TreeBasedGPGrammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+
+
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      return maxLen - 1;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    private Regex regex = new Regex("(?=ab)|(?=ba)"); // count the number of "ab" and "ba" pairs
+    public double Evaluate(string sentence) {
+      // sentence must contain only terminal symbols, we are not checking if the sentence is syntactically valid here because it would be too slow!
+      Debug.Assert(sentence.Any(c => grammar.IsTerminal(c)));
+      return regex.Matches(sentence).Count;
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      if (phrase.Length <= 1)
+        yield return new Feature("$$", 1.0);
+      else if (phrase.Length == 2)
+        yield return new Feature(phrase, 1.0);
+      else if (phrase.EndsWith("S")) // second to last symbol
+        yield return new Feature(phrase.Substring(phrase.Length - 3, 2), 1.0);
+      else // last symbol
+        yield return new Feature(phrase.Substring(phrase.Length - 2, 2), 1.0);
+
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPhraseSequenceProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPhraseSequenceProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalPhraseSequenceProblem.cs	(revision 12290)
@@ -0,0 +1,166 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Common;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // must find one of numCorrectPhrases*sequenceLen sequences where the quality of a sequence is the length of the subsequence containing only correct _phrases_ (of length phraseLen) and starting at the first position
+  // compared to the RoyalSequence problem this problem is harder because the number of different phrases starting at a position is much larger than the number of symbols (grows exponentially with the phrase-length)
+  // if phraseLen = 1 this is the same as the RoyalSequence problem
+  // parameters
+  // - alphabetSize: number of different symbols (max=26)
+  // - phraseLen: the length of a phrase in number of symbols
+  // - sequenceLen: the number of phrases in the correct subsequence (total sequence length is n * phraseLen
+  // - numCorrectPhrases: the number of correct phrases starting at each position
+  // - phrasesAsSets: switch to determine if the ordering of symbols within a phrase is relevant
+  // 
+  // this problem should be hard for GP and easy for MCTS (TD should not have an advantage compared to MCTS)
+  // for phraseLen > 1 this should be harder than RoyalSymbolProblem
+  // when phrases are symbol sets instead of sequences then value-estimation routines should be better (TD)
+  public class RoyalPhraseSequenceProblem : ISymbolicExpressionTreeProblem {
+
+    private readonly IGrammar grammar;
+    private readonly double correctReward;
+    private readonly double incorrectReward;
+    private readonly int sequenceLen;
+    private readonly int phraseLen;
+    private readonly bool phrasesAsSets;
+    private readonly SortedSet<string>[] optimalPhrasesForPos;
+    public string Name { get { return "RoyalPhraseSequence"; } }
+    public RoyalPhraseSequenceProblem(Random rand, int alphabetSize, int sequenceLen, int phraseLen = 1, int numCorrectPhrases = 1, double correctReward = 1.0, double incorrectReward = 0.0, bool phrasesAsSets = false) {
+      if (alphabetSize <= 0 || alphabetSize > 26) throw new ArgumentException();
+      if (sequenceLen <= 0) throw new ArgumentException();
+      if (numCorrectPhrases < 1 || numCorrectPhrases > alphabetSize) throw new ArgumentException();
+      if (phraseLen < 1) throw new ArgumentException();
+      if (correctReward <= incorrectReward) throw new ArgumentException();
+
+      this.sequenceLen = sequenceLen;
+      this.phraseLen = phraseLen;
+      this.correctReward = correctReward;
+      this.incorrectReward = incorrectReward;
+      this.phrasesAsSets = phrasesAsSets;
+
+      var sentenceSymbol = 'S';
+      var terminalSymbols = Enumerable.Range(0, alphabetSize).Select(off => (char)((byte)'a' + off)).ToArray();
+      var nonTerminalSymbols = new char[] { sentenceSymbol };
+
+      {
+        // create grammar
+        // S -> a..z | aS .. zS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t + sentenceSymbol.ToString())));
+
+        this.grammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+      {
+        // create grammar for tree-based GP
+        // S -> a..z | SS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(new Tuple<char, string>[] { Tuple.Create(sentenceSymbol, sentenceSymbol.ToString() + sentenceSymbol) });
+
+        this.TreeBasedGPGrammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+
+      this.optimalPhrasesForPos = new SortedSet<string>[sequenceLen];
+      for (int i = 0; i < sequenceLen; i++) {
+        optimalPhrasesForPos[i] = new SortedSet<string>();
+        for (int j = 0; j < numCorrectPhrases; j++) {
+          string phrase = "";
+          do {
+            for (int l = 0; l < phraseLen; l++) {
+              phrase += terminalSymbols.SelectRandom(rand);
+            }
+            phrase = CanonicalPhrase(phrase);
+          } while (optimalPhrasesForPos[i].Contains(phrase)); // don't allow duplicate phrases
+          optimalPhrasesForPos[i].Add(phrase);
+        }
+      }
+
+      Debug.Assert(Evaluate(BestKnownSolution) / BestKnownQuality(phraseLen * sequenceLen) == 1.0);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      return Math.Min(maxLen / phraseLen, sequenceLen) * correctReward; // integer division
+    }
+
+    public string BestKnownSolution {
+      get {
+        string solution = "";
+        for (int i = 0; i < sequenceLen; i++) {
+          solution += optimalPhrasesForPos[i].First();
+        }
+        return solution;
+      }
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      // sentence must contain only terminal symbols, we are not checking if the sentence is syntactically valid here because it would be too slow!
+      Debug.Assert(sentence.Any(c => grammar.IsTerminal(c)));
+      // as long as only correct symbols are found we increase the reward by +1
+      // on the first incorrect symbol we return
+      var reward = 0.0;
+      for (int i = 0; i < Math.Min(sentence.Length / phraseLen, sequenceLen); i++) {
+        var canonicalPhrase = CanonicalPhrase(sentence.Substring(i * phraseLen, phraseLen));
+        if (optimalPhrasesForPos[i].Contains(canonicalPhrase)) {
+          reward += correctReward;
+        } else {
+          // alternatively reduce reward by number of remaining phrases
+          return Math.Max(0.0, reward + incorrectReward * (sentence.Length / phraseLen - i));
+          // stop on first incorrect symbol and return reward
+          //return reward;
+        }
+      }
+      return reward;
+    }
+
+    // TODO: cache canonical phrases in most-recently used dictionary for increased performance (see symbolicregressionpoly10problem)
+    private string CanonicalPhrase(string phrase) {
+      if (phrasesAsSets) return string.Join("", phrase.OrderBy(ch => (byte)ch));
+      else return phrase;
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      if (phrasesAsSets) {
+        var sb = new StringBuilder();
+        var numPhrases = phrase.Length / phraseLen;
+        for (int phraseIdx = 0; phraseIdx < numPhrases; phraseIdx++) {
+          var sentenceIdx = phraseIdx * phraseLen;
+          var subphrase = phrase.Substring(sentenceIdx, phraseLen);
+          subphrase = CanonicalPhrase(subphrase);
+          sb.Append(subphrase);
+        }
+
+        var remainder = phrase.Substring(numPhrases * phraseLen, phrase.Length - (numPhrases * phraseLen));
+        remainder = CanonicalPhrase(remainder);
+        sb.Append(remainder);
+
+        return sb.ToString();
+      } else
+        return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase)
+    {
+      return new Feature[] {new Feature(phrase, 1.0)};
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalRoadProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalRoadProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalRoadProblem.cs	(revision 12290)
@@ -0,0 +1,41 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class RoyalRoadProblem : IProblem {
+    // inspired by Mitchell, Forrest, Holland: "The Royal Road for Genetic Algorithms: Fitness Landscapes and GA Performance", 1992
+    private const string grammarString = @"
+G(S):
+S -> 1 | 0 | 1S | 0S
+";
+
+    private readonly IGrammar grammar;
+    public string Name { get { return "RoyalRoad"; } }
+    public RoyalRoadProblem() {
+      this.grammar = new Grammar(grammarString);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      // for now only an upper bound is returned, ideally all fitness cases are predicted correctly
+      throw new NotImplementedException();
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      throw new NotImplementedException();
+    }
+    public string CanonicalRepresentation(string phrase) {
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase)
+    {
+      throw new NotImplementedException();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSequenceProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSequenceProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSequenceProblem.cs	(revision 12290)
@@ -0,0 +1,117 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Common;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // must find one of k*sequenceLen sequences where the quality of a sequence is the length of the subsequence containing only correct symbols and starting at the first symbol
+  // parameters
+  // - alphabetSize: number of different symbols (max=26)
+  // - sequenceLen: length of the correct subsequence
+  // - k: the number of correct symbols at each position
+  // 
+  // this problem should be hard for GP and easy for MCTS (TD should not have an advantage compared to MCTS)
+  public class RoyalSequenceProblem : ISymbolicExpressionTreeProblem {
+
+    private readonly IGrammar grammar;
+    private readonly double correctReward;
+    private readonly double incorrectReward;
+    private readonly int sequenceLen;
+    private readonly SortedSet<char>[] optimalSymbolsForPos;
+    public string Name { get { return "RoyalSequence"; } }
+    public RoyalSequenceProblem(Random rand, int alphabetSize, int sequenceLen, int k = 1, double correctReward = 1.0, double incorrectReward = 0.0) {
+      if (alphabetSize <= 0 || alphabetSize > 26) throw new ArgumentException();
+      if (sequenceLen <= 0) throw new ArgumentException();
+      if (k < 1 || k > alphabetSize) throw new ArgumentException();
+      if (correctReward <= incorrectReward) throw new ArgumentException();
+      this.sequenceLen = sequenceLen;
+      this.correctReward = correctReward;
+      this.incorrectReward = incorrectReward;
+
+      const char sentenceSymbol = 'S';
+      var terminalSymbols = Enumerable.Range(0, alphabetSize).Select(off => (char)((byte)'a' + off)).ToArray();
+      var nonTerminalSymbols = new char[] { sentenceSymbol };
+
+      {
+        // create grammar for sequential search
+        // S -> a..z | aS .. zS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t + sentenceSymbol.ToString())));
+        this.grammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+      {
+        // create grammar for sequential search
+        // S -> a..z | SS
+        var rules = terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t.ToString()))
+          .Concat(terminalSymbols.Select(t => Tuple.Create(sentenceSymbol, t + sentenceSymbol.ToString())));
+        this.grammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols, rules);
+      }
+
+      this.optimalSymbolsForPos = new SortedSet<char>[sequenceLen];
+      for (int i = 0; i < sequenceLen; i++) {
+        optimalSymbolsForPos[i] = new SortedSet<char>();
+        for (int j = 0; j < k; j++) {
+          char ch;
+          do {
+            ch = terminalSymbols.SelectRandom(rand);
+          } while (optimalSymbolsForPos[i].Contains(ch));
+          optimalSymbolsForPos[i].Add(ch);
+        }
+      }
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      return Math.Min(maxLen, sequenceLen) * correctReward;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      // sentence must contain only terminal symbols, we are not checking if the sentence is syntactically valid here because it would be too slow!
+      Debug.Assert(sentence.Any(c => grammar.IsTerminal(c)));
+      var reward = 0.0;
+      for (int i = 0; i < Math.Min(sentence.Length, sequenceLen); i++) {
+        if (optimalSymbolsForPos[i].Contains(sentence[i])) {
+          reward += correctReward;
+        } else {
+          //  reduce reward by number of remaining symbols
+          return Math.Max(0.0, reward + incorrectReward * (sentence.Length - i));
+        }
+      }
+      return reward;
+    }
+
+    // in each position there could be multiple correct and incorrect symbols
+    public string CanonicalRepresentation(string phrase) {
+      var sb = new StringBuilder();
+      for (int i = 0; i < phrase.Length; i++) {
+        if (optimalSymbolsForPos[i].Contains(phrase[i])) {
+          sb.Append(optimalSymbolsForPos[i].First()); // all symbols in the set are equivalent
+        } else {
+          sb.Append(phrase[i]);
+        }
+      }
+      return sb.ToString();
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      throw new NotImplementedException();
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSymbolProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSymbolProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalSymbolProblem.cs	(revision 12290)
@@ -0,0 +1,64 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // counts the number of times a symbol occurs in a sentence
+  public class RoyalSymbolProblem : ISymbolicExpressionTreeProblem {
+    private const string grammarString = @"
+G(S):
+S -> a | aS | b | bS
+";
+
+    private const string hlGrammarString = @"
+G(S):
+S -> a | b | SS
+";
+
+    // Obj(s \in L(G(S))) = Anzahl "a"-Symbole in s
+
+    private readonly IGrammar grammar;
+    public string Name { get { return "RoyalSymbol"; } }
+    public RoyalSymbolProblem() {
+      this.grammar = new Grammar(grammarString);
+      this.TreeBasedGPGrammar = new Grammar(hlGrammarString);
+      //TODO: allow configuration of the number of symbols
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      return maxLen;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    private Regex regex = new Regex("a"); // count the number of "a"s
+    public double Evaluate(string sentence) {
+      // sentence must contain only terminal symbols, we are not checking if the sentence is syntactically valid here because it would be too slow!
+      Debug.Assert(sentence.Any(c => grammar.IsTerminal(c)));
+      return regex.Matches(sentence).Count;
+    }
+    public string CanonicalRepresentation(string phrase) {
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      throw new NotImplementedException();
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name);
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalTreeProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalTreeProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/RoyalTreeProblem.cs	(revision 12290)
@@ -0,0 +1,276 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class RoyalTreeProblem : ISymbolicExpressionTreeProblem {
+    // Punch: "How Effective Are Multiple Populations in Genetic Programming", 1998
+    // In fact, for a new GP system it is dicult to judge whether it is performing
+    // as intended or not, since the programs it generates are not necessarily identical to
+    // those generated by other GP systems. This raised two questions: what constitutes
+    // a "standard" problem in GP, and how do we rate the performance of a system on
+    // such a problem.
+    // One of the goals of this research was to create a benchmark problem to test how
+    // well a particular GP configuration would perform as compared to other configurations.
+    // ...
+    // We were interested in addressing the same issues, showing how GP could address
+    // difficult problems as well as providing a tunable benchmark for comparing GP con-
+    // figurations. Furthermore, we were interested in creating and using this benchmark
+    // to test the effectiveness of coarse-grain parallel GP's as compared to single population
+    // GP's.
+    // ...
+    // A suitable benchmark problem which has only a single, unique tree as an answer can remove this obstruction.
+
+    private readonly IGrammar grammar;
+    private readonly char targetRootSymbol;
+    public string Name { get { return "RoyalTree"; } }
+    // the number of levels determines the number of non-terminal symbols
+    // non-terminal A has only one argument, non-terminal B has two arguments ...
+    // optimal level-e tree has quality 122880, 
+    // level-a (a) tree has 
+    // level-f (6) tree "is extremely difficult"
+    // level-g (7) tree "never succeeded"
+    public RoyalTreeProblem(int levels) {
+      if (levels < 1 || levels > 7) throw new ArgumentException();
+      var sentenceSymbol = 'S';
+      var terminalSymbols = new char[] { 'x', 'y', 'z' };
+      // originally non-terminal-symbols (but for the sequential search grammar these are only alternatives
+      var nonTerminalSymbols = Enumerable.Range(0, levels).Select(lvl => (char)(lvl + (byte)'A')).ToArray(); // [A, B ... ]
+      var arities = Enumerable.Range(1, levels).ToArray(); // [1, 2, ... levels]
+      this.targetRootSymbol = char.ToLower(nonTerminalSymbols.Last());
+      {
+        // grammar for sequential search
+        // S -> x | y | z | aS | bSS | cSSS ...
+
+        var rules = new List<Tuple<char, string>>();
+        foreach (var symb in terminalSymbols) {
+          rules.Add(Tuple.Create('S', symb.ToString()));
+        }
+        for (int ntIdx = 0; ntIdx < nonTerminalSymbols.Length; ntIdx++) {
+          var opSymb = char.ToLower(nonTerminalSymbols[ntIdx]);
+          var remChain = string.Join("", Enumerable.Repeat('S', arities[ntIdx]));
+          rules.Add(Tuple.Create('S', opSymb + remChain));
+        }
+
+        this.grammar = new Grammar(sentenceSymbol, terminalSymbols.Concat(nonTerminalSymbols.Select(char.ToLower)), new char[] { 'S' }, rules);
+      }
+
+      {
+        // grammar for tree-based GP
+        // S -> x | y | z | A | B | C ...
+        // A -> S
+        // B -> SS
+        // C -> SSS
+        var rules = new List<Tuple<char, string>>();
+        foreach (var symb in terminalSymbols.Concat(nonTerminalSymbols)) {
+          rules.Add(Tuple.Create('S', symb.ToString()));
+        }
+
+        for (int ntIdx = 0; ntIdx < nonTerminalSymbols.Length; ntIdx++) {
+          var ntSymb = nonTerminalSymbols[ntIdx];
+          var alt = string.Join("", Enumerable.Repeat('S', arities[ntIdx]));
+          rules.Add(Tuple.Create(ntSymb, alt));
+        }
+        this.TreeBasedGPGrammar = new Grammar(sentenceSymbol, terminalSymbols, nonTerminalSymbols.Concat(new char[] { 'S' }), rules);
+      }
+
+      maxSizeForLevel = new int[8];
+      optimalSentencesForLevel = new string[8];
+      optimalQualityForLevel = new double[8];
+      maxSizeForLevel[0] = 1; // lvl-0 (x | y)
+      maxSizeForLevel[1] = 2; // lvl-1 (e.g. ax)
+      optimalSentencesForLevel[0] = "x";
+      optimalQualityForLevel[0] = 1;
+      //optimalSentencesForLevel[1] = "ax";
+      //optimalQualityForLevel[1] = 4;
+
+      for (int i = 1; i < levels; i++) {
+        maxSizeForLevel[i] = i * maxSizeForLevel[i - 1] + 1; // e.g. lvl-2: baxax
+
+        var opSymb = char.ToLower(nonTerminalSymbols[i - 1]);
+        optimalSentencesForLevel[i] = opSymb +
+                                       string.Join("",
+                                         Enumerable.Repeat(optimalSentencesForLevel[i - 1], arities[i - 1]));
+
+        optimalQualityForLevel[i] = Evaluate(optimalSentencesForLevel[i]);
+      }
+
+      // from the paper
+      Debug.Assert(maxSizeForLevel[5] == 326);
+      Debug.Assert(maxSizeForLevel[6] == 1957);
+    }
+
+    private readonly string[] optimalSentencesForLevel;
+    private readonly double[] optimalQualityForLevel;
+    private readonly int[] maxSizeForLevel;
+
+    public double BestKnownQuality(int maxLen) {
+      // search for corresponding level
+      int level = 0;
+      while (level+1 < maxSizeForLevel.Length && maxSizeForLevel[level + 1] < maxLen) level++;
+      // ASSERT: maxSizeForLevel[level + 1] >= maxLen
+      return optimalQualityForLevel[level];
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      int pos = 0;
+      bool perfect;
+      return Evaluate(sentence, ref pos, out perfect);
+    }
+
+    private double Evaluate(string sentence, ref int pos, out bool perfect) {
+      const double completeBonus = 2.0;
+      double t = 0.0;
+      double weight = 0.0;
+      double sum = 0.0;
+      bool correctRoot = false;
+      bool allPerfect = true, tPerfect;
+      int arity;
+      switch (sentence[pos]) {
+        case 'a':
+          pos++;
+          correctRoot = (sentence[pos] == 'x');
+          t = Evaluate(sentence, ref pos, out tPerfect); // ignores the perfect flag (false for terminals)
+          weight = GetWeight(correctRoot, true);
+          sum = weight * t;
+          perfect = correctRoot;
+          if (correctRoot) sum *= completeBonus;
+          return sum;
+        case 'b': {
+            arity = 2;
+            pos++;
+            for (int i = 0; i < arity; i++) {
+              correctRoot = (sentence[pos] == 'a');
+              t = Evaluate(sentence, ref pos, out tPerfect);
+              weight = GetWeight(correctRoot, tPerfect);
+              allPerfect &= correctRoot & tPerfect;
+              sum += weight * t;
+            }
+            perfect = allPerfect;
+            if (allPerfect) sum *= completeBonus;
+            return sum;
+          }
+        case 'c': {
+            arity = 3;
+            sum = 0.0;
+            pos++;
+            for (int i = 0; i < arity; i++) {
+              correctRoot = (sentence[pos] == 'b');
+              t = Evaluate(sentence, ref pos, out tPerfect);
+              weight = GetWeight(correctRoot, tPerfect);
+              allPerfect &= correctRoot & tPerfect;
+              sum += weight * t;
+            }
+            perfect = allPerfect;
+            if (allPerfect) sum *= completeBonus;
+            return sum;
+          }
+        case 'd': {
+            arity = 4;
+            sum = 0.0;
+            pos++;
+            for (int i = 0; i < arity; i++) {
+              correctRoot = (sentence[pos] == 'c');
+              t = Evaluate(sentence, ref pos, out tPerfect);
+              weight = GetWeight(correctRoot, tPerfect);
+              allPerfect &= correctRoot & tPerfect;
+              sum += weight * t;
+            }
+            perfect = allPerfect;
+            if (allPerfect) sum *= completeBonus;
+            return sum;
+          }
+        case 'e': {
+            arity = 5;
+            sum = 0.0;
+            pos++;
+            for (int i = 0; i < arity; i++) {
+              correctRoot = (sentence[pos] == 'd');
+              t = Evaluate(sentence, ref pos, out tPerfect);
+              weight = GetWeight(correctRoot, tPerfect);
+              allPerfect &= correctRoot & tPerfect;
+              sum += weight * t;
+            }
+            perfect = allPerfect;
+            if (allPerfect) sum *= completeBonus;
+            return sum;
+          }
+        case 'f': {
+            arity = 6;
+            sum = 0.0;
+            pos++;
+            for (int i = 0; i < arity; i++) {
+              correctRoot = (sentence[pos] == 'e');
+              t = Evaluate(sentence, ref pos, out tPerfect);
+              weight = GetWeight(correctRoot, tPerfect);
+              allPerfect &= correctRoot & tPerfect;
+              sum += weight * t;
+            }
+            perfect = allPerfect;
+            if (allPerfect) sum *= completeBonus;
+            return sum;
+          }
+        case 'g': {
+            arity = 7;
+            sum = 0.0;
+            pos++;
+            for (int i = 0; i < arity; i++) {
+              correctRoot = (sentence[pos] == 'f');
+              t = Evaluate(sentence, ref pos, out tPerfect);
+              weight = GetWeight(correctRoot, tPerfect);
+              allPerfect &= correctRoot & tPerfect;
+              sum += weight * t;
+            }
+            perfect = allPerfect;
+            if (allPerfect) sum *= completeBonus;
+            return sum;
+          }
+        case 'x':
+          pos++;
+          perfect = false;
+          return 1.0;
+        case 'y':
+        case 'z':
+          pos++;
+          perfect = false;
+          return 0.0;
+        default: throw new ArgumentException();
+      }
+    }
+
+    private double GetWeight(bool correctRoot, bool perfect) {
+      const double fullBonus = 2.0;
+      const double partialBonus = 1.0;
+      const double penalty = 0.33;
+      if (correctRoot && perfect) return fullBonus;
+      else if (correctRoot) return partialBonus;
+      else return penalty;
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      throw new NotImplementedException();
+      return phrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      throw new NotImplementedException();
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      foreach (var s in tree.Root.GetSubtree(0).GetSubtree(0).IterateNodesPrefix()) {
+        if (s.Symbol.Name == "S") continue;
+        sb.Append(s.Symbol.Name.ToLower());
+      }
+      return sb.ToString();
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/SantaFeAntProblem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/SantaFeAntProblem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/SantaFeAntProblem.cs	(revision 12290)
@@ -0,0 +1,303 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Runtime.Remoting.Messaging;
+using System.Text;
+using System.Text.RegularExpressions;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class SantaFeAntProblem : ISymbolicExpressionTreeProblem {
+    private const string grammarString = @"
+G(A):
+A -> l | r | m | ?(A)(A) | lA | rA | mA 
+";
+    // for tree-based GP in HL we need a different grammar for the same language
+    // A = Ant, P = Prog2 and Prog3, I = IfFoodAhead
+    private const string hlGrammarString = @"
+    G(A):
+    A -> l | r | m | P | I
+    P -> AA | AAA
+    I -> AA
+    ";
+
+
+    // original koza grammar
+    // Ant -> left | right | move | if-food-ahead Ant Ant | Ant Ant | Ant Ant Ant
+    // 
+    // here we use a modified grammar which has only one syntax tree for each sentence
+
+
+    private readonly IGrammar grammar;
+    public string Name { get { return "SantaFeAnt"; } }
+
+    public SantaFeAntProblem() {
+      this.grammar = new Grammar(grammarString);
+      this.TreeBasedGPGrammar = new Grammar(hlGrammarString);
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      // for now only an upper bound is returned, ideally all food pellets are discovered
+      return 89;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      var ant = new Ant();
+      int p = 0;
+      Run(ant, sentence, ref p);
+      return ant.FoodEaten;
+    }
+
+    private static void Run(Ant ant, string sentence, ref int p, bool stopAfterFirst = false) {
+      while (!ant.Done()) {
+        if (p >= sentence.Length) {
+          if (stopAfterFirst) return;
+          p = 0; // restart
+        }
+        switch (sentence[p]) {
+          case 'r': {
+              ant.TurnRight();
+              p++;
+              break;
+            }
+          case 'l': {
+              ant.TurnLeft();
+              p++;
+              break;
+            }
+          case 'm': {
+              ant.Move();
+              p++;
+              break;
+            }
+          case '?': {
+              p++; // skip p
+              if (ant.IsFoodAhead()) {
+                if (sentence[p] != '(') throw new ArgumentException();
+                p++;
+                Run(ant, sentence, ref p);      // it cannot happen that we run over the the end of the sentence here
+                if (ant.Done()) return;
+                if (sentence[p] != '(') throw new ArgumentException();
+                p++;
+                Skip(sentence, ref p);
+              } else {
+                if (sentence[p] != '(') throw new ArgumentException();
+                p++;
+                Skip(sentence, ref p);
+                if (sentence[p] != '(') throw new ArgumentException();
+                p++;
+                Run(ant, sentence, ref p);    // it cannot happen that we run over the the end of the sentence here
+              }
+              break;
+            }
+          case '.': {
+              // nop
+              p++;
+              ant.Nop();
+              break;
+            }
+          case ')': {
+              p++; // skip
+              return;     // caller must continue processing
+            }
+          default: {
+              throw new ArgumentException();
+            }
+        }
+      }
+    }
+
+    private static void Skip(string sentence, ref int p) {
+      int openCount = 1;
+      while (openCount > 0) {
+        if (sentence[p] == '(') openCount++;
+        else if (sentence[p] == ')') openCount--;
+        p++;
+      }
+    }
+
+    public string CanonicalRepresentation(string phrase) {
+      //phrase = phrase.Replace("A", ".");
+      var sb = new StringBuilder(phrase);
+      string canonicalPhrase = phrase;
+      string oldPhrase;
+      do {
+        oldPhrase = canonicalPhrase;
+        sb.Replace("ll", "rr").Replace("rl", "").Replace("lr", "").Replace("lll", "r").Replace("rrr", "l").Replace("?()()", "");
+        //sb.Replace("?(m)(m)", "?()()m").Replace("?(l)(l)", "?()()l").Replace("?(r)(r)", "?()()r");
+        canonicalPhrase = sb.ToString();
+      } while (canonicalPhrase != oldPhrase);
+      return canonicalPhrase;
+    }
+
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      yield return new Feature(phrase, 1.0);
+      //var ant = new Ant(false);
+      //int p = 0;
+      //Run(ant, phrase.Replace('A', '.'), ref p, true);
+      //yield return new Feature(ant.PosX + "x" + ant.PosY + "-" + ant.Heading, 1.0);
+    }
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+      ConvertTreeToSentence(tree.Root.GetSubtree(0).GetSubtree(0), sb);
+      return sb.ToString();
+    }
+    private void ConvertTreeToSentence(ISymbolicExpressionTreeNode node, StringBuilder sb) {
+      if (node.SubtreeCount == 0) {
+        // terminal
+        sb.Append(node.Symbol.Name);
+      } else if (node.Symbol.Name == "I") {
+        Debug.Assert(node.SubtreeCount == 2);
+        sb.Append("?(");
+        ConvertTreeToSentence(node.GetSubtree(0), sb);
+        sb.Append(")(");
+        ConvertTreeToSentence(node.GetSubtree(1), sb);
+        sb.Append(")");
+      } else {
+        foreach (var subTree in node.Subtrees) { ConvertTreeToSentence(subTree, sb); }
+      }
+    }
+  }
+
+  public class Ant {
+    private int maxSteps = 600;
+    public int MaxSteps { get { return maxSteps; } set { maxSteps = value; } }
+    public enum HeadingEnum { North, East, South, West };
+    public int FoodEaten { get; private set; }
+    private readonly char[][] world = new char[32][];
+    private int posX;
+    private int posY;
+    private int steps;
+    private HeadingEnum heading;
+    public int PosX { get { return posX; } }
+    public int PosY { get { return posY; } }
+    public HeadingEnum Heading { get { return heading; } }
+    private bool recordTrail = false;
+    private StringBuilder trailBuilder;
+
+    public string Trail {
+      get {
+        if (!recordTrail) throw new NotSupportedException();
+        else return trailBuilder.ToString() + heading; // add final heading as state
+      }
+    }
+
+    public Ant(bool recordTrail = false) {
+      world[00] = " ###                            ".ToCharArray();
+      world[01] = "   #                            ".ToCharArray();
+      world[02] = "   #                    .###..  ".ToCharArray();
+      world[03] = "   #                    #    #  ".ToCharArray();
+      world[04] = "   #                    #    #  ".ToCharArray();
+      world[05] = "   ####.#####       .##..    .  ".ToCharArray();
+      world[06] = "            #       .        #  ".ToCharArray();
+      world[07] = "            #       #        .  ".ToCharArray();
+      world[08] = "            #       #        .  ".ToCharArray();
+      world[09] = "            #       #        #  ".ToCharArray();
+      world[10] = "            .       #        .  ".ToCharArray();
+      world[11] = "            #       .        .  ".ToCharArray();
+      world[12] = "            #       .        #  ".ToCharArray();
+      world[13] = "            #       #        .  ".ToCharArray();
+      world[14] = "            #       #  ...###.  ".ToCharArray();
+      world[15] = "            .   .#...  #        ".ToCharArray();
+      world[16] = "            .   .      .        ".ToCharArray();
+      world[17] = "            #   .      .        ".ToCharArray();
+      world[18] = "            #   #      .#...    ".ToCharArray();
+      world[19] = "            #   #          #    ".ToCharArray();
+      world[20] = "            #   #          .    ".ToCharArray();
+      world[21] = "            #   #          .    ".ToCharArray();
+      world[22] = "            #   .      ...#.    ".ToCharArray();
+      world[23] = "            #   .      #        ".ToCharArray();
+      world[24] = " ..##..#####.   #               ".ToCharArray();
+      world[25] = " #              #               ".ToCharArray();
+      world[26] = " #              #               ".ToCharArray();
+      world[27] = " #     .#######..               ".ToCharArray();
+      world[28] = " #     #                        ".ToCharArray();
+      world[29] = " .     #                        ".ToCharArray();
+      world[30] = " .####..                        ".ToCharArray();
+      world[31] = "                                ".ToCharArray();
+
+      posX = 0;
+      posY = 0;
+      heading = HeadingEnum.East;
+      FoodEaten = 0;
+      steps = 0;
+
+      this.recordTrail = recordTrail;
+      if (this.recordTrail) trailBuilder = new StringBuilder();
+    }
+
+    public bool Done() {
+      return steps > maxSteps;
+    }
+
+    public void TurnRight() {
+      if (steps <= maxSteps) {
+        steps++;
+        if (heading == HeadingEnum.West) heading = HeadingEnum.North;
+        else heading++;
+      }
+    }
+
+    public void TurnLeft() {
+      if (steps <= maxSteps) {
+        steps++;
+        if (heading == HeadingEnum.North) heading = HeadingEnum.West;
+        else heading--;
+      }
+    }
+
+    public void Move() {
+      if (steps <= maxSteps) {
+        steps++;
+        MoveInternal(ref posX, ref posY);
+        if (world[posY][posX] == '#') {
+          FoodEaten++;
+          world[posY][posX] = '.';
+        }
+        if (recordTrail) trailBuilder.Append("m" + posX + "x" + posY); // record position change
+      }
+    }
+
+    public void Nop() {
+      // wait one time step
+      if (steps <= maxSteps) {
+        steps++;
+      }
+    }
+
+    private void MoveInternal(ref int posX, ref int posY) {
+      switch (heading) {
+        case HeadingEnum.North:
+          posY = (posY + 31) % 32;
+          break;
+        case HeadingEnum.East:
+          posX = (posX + 1) % 32;
+          break;
+        case HeadingEnum.South:
+          posY = (posY + 1) % 32;
+          break;
+        case HeadingEnum.West:
+          posX = (posX + 31) % 32;
+          break;
+      }
+    }
+
+    public bool IsFoodAhead() {
+      int nextPosX = posX;
+      int nextPosY = posY;
+      MoveInternal(ref nextPosX, ref nextPosY);
+
+      if (recordTrail) trailBuilder.Append("?" + nextPosX + "x" + nextPosY); // record check
+
+      return world[nextPosY][nextPosX] == '#';
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/SymbolicRegressionPoly10Problem.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/SymbolicRegressionPoly10Problem.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Problems/SymbolicRegressionPoly10Problem.cs	(revision 12290)
@@ -0,0 +1,208 @@
+﻿using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Diagnostics;
+using System.Linq;
+using System.Net;
+using System.Security;
+using System.Security.AccessControl;
+using System.Text;
+using HeuristicLab.Common;
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class SymbolicRegressionPoly10Problem : ISymbolicExpressionTreeProblem {
+    //    private const string grammarString = @"
+    //    G(E):
+    //    E -> V | V+E | V-E | V*E | (E)
+    //    V -> a .. j 
+    //    ";
+    //private const string grammarString = @"
+    //G(E):
+    //E -> a | b | c | d | e | f | g | h | i | j | a+E | b+E | c+E | d+E | e+E | f+E | g+E | h+E | i+E | j+E | a*E | b*E | c*E | d*E | e*E | f*E | g*E | h*E | i*E | j*E 
+    //";
+    private const string grammarString = @"
+    G(E):
+    E -> a | b | c | d | e | f | g | h | i | j | a+E | b+E | c+E | d+E | e+E | f+E | g+E | h+E | i+E | j+E | a*E | b*E | c*E | d*E | e*E | f*E | g*E | h*E | i*E | j*E  
+    ";
+
+    // for tree-based GP in HL we need a different grammar for the same language
+    // E = expr, S = sum, P = product
+    private const string hlGrammarString = @"
+    G(E):
+    E -> S | P | a | b | c | d | e | f | g | h | i | j
+    S -> EE | EEE
+    P -> EE | EEE
+    ";
+    // mininmal tree for the optimal expr (40 nodes)
+    // E S
+    //     E S
+    //         E P 
+    //           E a E b
+    //         E P 
+    //           E c E d
+    //         E P 
+    //           E e E f
+    //     E S
+    //         E P 
+    //           E a E g E i
+    //         E P 
+    //           E c E f E j
+
+    public IGrammar TreeBasedGPGrammar { get; private set; }
+
+    private readonly IGrammar grammar;
+
+    private readonly int N;
+    private readonly double[][] x;
+    private readonly double[] y;
+    public string Name { get { return "SymbolicRegressionPoly10"; } }
+
+    public SymbolicRegressionPoly10Problem() {
+      this.grammar = new Grammar(grammarString);
+      this.TreeBasedGPGrammar = new Grammar(hlGrammarString);
+
+      this.N = 500;
+      this.x = new double[N][];
+      this.y = new double[N];
+
+      GenerateData();
+    }
+
+    private void GenerateData() {
+      // generate data with fixed seed to make sure that data is always the same
+      var rand = new Random(31415);
+      for (int i = 0; i < N; i++) {
+        x[i] = new double[10];
+        for (int j = 0; j < 10; j++) {
+          x[i][j] = rand.NextDouble() * 2 - 1;
+        }
+        // poly-10 no noise
+        /* a*b + c*d + e*f + a*g*i + c*f*j */
+        y[i] = x[i][0] * x[i][1] +
+               x[i][2] * x[i][3] +
+               x[i][4] * x[i][5] +
+               x[i][0] * x[i][6] * x[i][8] +
+               x[i][2] * x[i][5] * x[i][9];
+      }
+    }
+
+    public double BestKnownQuality(int maxLen) {
+      // for now only an upper bound is returned, ideally we have an R² of 1.0
+      // the optimal R² can only be reached for sentences of at least 23 symbols
+      return 1.0;
+    }
+
+    public IGrammar Grammar {
+      get { return grammar; }
+    }
+
+    public double Evaluate(string sentence) {
+      var interpreter = new ExpressionInterpreter();
+      return HeuristicLab.Common.Extensions.RSq(y, Enumerable.Range(0, N).Select(i => interpreter.Interpret(sentence, x[i])).ToArray());
+    }
+
+
+    // most-recently-used caching (with limited capacity) for canonical representations
+    MostRecentlyUsedCache<string, string> canonicalPhraseCache = new MostRecentlyUsedCache<string, string>(100000);
+    // right now only + and * is supported
+    public string CanonicalRepresentation(string phrase) {
+      string canonicalPhrase;
+      if (!canonicalPhraseCache.TryGetValue(phrase, out canonicalPhrase)) {
+        var terms = phrase.Split('+');
+        var canonicalTerms = new SortedSet<string>();
+        // only the last term might contain a NT-symbol. make sure this term is added at the end
+        for (int i = 0; i < terms.Length - 1; i++) {
+          canonicalTerms.Add(CanonicalTerm(terms[i]));
+        }
+
+        var sb = new StringBuilder(phrase.Length);
+        foreach (var t in canonicalTerms)
+          sb.Append(t).Append('+');
+
+        sb.Append(CanonicalTerm(terms[terms.Length - 1]));
+        canonicalPhrase = sb.ToString();
+        canonicalPhraseCache.Add(phrase, canonicalPhrase);
+      }
+      return canonicalPhrase;
+    }
+
+    // cache the canonical form of terms for performance reasons
+    private Dictionary<string, string> canonicalTermDictionary = new Dictionary<string, string>();
+    private string CanonicalTerm(string term) {
+      string canonicalTerm;
+      if (!canonicalTermDictionary.TryGetValue(term, out canonicalTerm)) {
+        // add
+        var chars = term.ToCharArray();
+        Array.Sort(chars);
+        var sb = new StringBuilder(chars.Length);
+        // we want to have the up-case characters last
+        for (int i = chars.Length - 1; i > 0; i--) {
+          if (chars[i] != '*') {
+            sb.Append(chars[i]);
+            if (chars[i - 1] != '*') sb.Append('*');
+          }
+        }
+        if (chars[0] != '*') sb.Append(chars[0]); // last term
+        canonicalTerm = sb.ToString();
+        canonicalTermDictionary.Add(term, canonicalTerm);
+      }
+      return canonicalTerm;
+    }
+
+    private double[] varIds = new double[] { };
+
+    // splits the phrase into terms and creates (sparse) term-occurrance features
+    public IEnumerable<Feature> GetFeatures(string phrase) {
+      // var canonicalTerms = new HashSet<string>();
+      // foreach (string t in phrase.Split('+')) {
+      //   canonicalTerms.Add(CanonicalTerm(t));
+      // }
+      // return canonicalTerms.Select(entry => new Feature(entry, 1.0))
+      //   .Concat(new Feature[] { new Feature(CanonicalRepresentation(phrase), 1.0) });
+
+      var partialInterpreter = new PartialExpressionInterpreter();
+      var vars = new double[] { 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, };
+      var s = partialInterpreter.Interpret(phrase, vars);
+      //if (s.Any())
+      //  return new Feature[] { new Feature(s.Pop().ToString(), 1.0), };
+      //else
+      //  return new Feature[] { new Feature("$", 1.0), };
+      return new Feature[] { new Feature(string.Join(",", s), 1.0) };
+    }
+
+    public string ConvertTreeToSentence(ISymbolicExpressionTree tree) {
+      var sb = new StringBuilder();
+
+      TreeToSentence(tree.Root.GetSubtree(0).GetSubtree(0), sb);
+
+      return sb.ToString();
+    }
+
+    private void TreeToSentence(ISymbolicExpressionTreeNode treeNode, StringBuilder sb) {
+      if (treeNode.SubtreeCount == 0) {
+        // terminal
+        sb.Append(treeNode.Symbol.Name);
+      } else {
+        string op = string.Empty;
+        switch (treeNode.Symbol.Name) {
+          case "S": op = "+"; break;
+          case "P": op = "*"; break;
+          default: {
+              Debug.Assert(treeNode.SubtreeCount == 1);
+              break;
+            }
+        }
+        // nonterminal
+        if (op == "+") sb.Append("(");
+        TreeToSentence(treeNode.Subtrees.First(), sb);
+        foreach (var subTree in treeNode.Subtrees.Skip(1)) {
+          sb.Append(op);
+          TreeToSentence(subTree, sb);
+        }
+        if (op == "+") sb.Append(")");
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Problems.GrammaticalOptimization")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("HeuristicLab.Problems.GrammaticalOptimization")]
+[assembly: AssemblyCopyright("Copyright © Microsoft 2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("3610cf30-7829-4163-bac6-46718baf5e80")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/ReadonlySequence.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/ReadonlySequence.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/ReadonlySequence.cs	(revision 12290)
@@ -0,0 +1,97 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class ReadonlySequence : Sequence {
+    private int symbolsOffset = 0; // the sequence does not have to start with the first symbol of the symbols array (when we reuse these arrays)
+
+    // cloning constructor for readonly sequences
+    // does not allocate the symbols array of the base class
+    // instead: reuse the symbols array (both sequences are readonly)
+    private ReadonlySequence(ReadonlySequence original)
+      : base() {
+      base.symbols = original.symbols;
+      this.symbolsOffset = original.symbolsOffset;
+    }
+    public ReadonlySequence(string s)
+      : base(s, s.Length) {
+    }
+
+    public ReadonlySequence(char ch)
+      : base(ch, 1) {
+    }
+
+    public ReadonlySequence(Sequence s)
+      : base(s, s.Length) {
+    }
+
+    public override void ReplaceAt(int position, int len, Sequence replacement) {
+      throw new NotSupportedException();
+    }
+
+    public override char this[int idx] {
+      get {
+        return base[idx + symbolsOffset];
+      }
+      set {
+        throw new NotSupportedException();
+      }
+    }
+
+    public override IEnumerator<char> GetEnumerator() {
+      return symbols.Skip(symbolsOffset).Take(Length).GetEnumerator();
+    }
+
+    public override string ToString() {
+      var sb = new StringBuilder(Length);
+      sb.Append(symbols, symbolsOffset, Length);
+      return sb.ToString();
+    }
+
+    public new ReadonlySequence Subsequence(int startIdx, int len) {
+      if (startIdx < 0 || len < 0) throw new ArgumentException();
+      if (startIdx >= this.Length) throw new ArgumentException();
+      if (startIdx + len > this.Length) throw new ArgumentException();
+      var subsequence = new ReadonlySequence(this) { symbolsOffset = startIdx + this.symbolsOffset, Length = len };
+
+
+      if (FirstNonTerminalIndex < 0) {
+        subsequence.FirstNonTerminalIndex = -1;
+      } else if (FirstNonTerminalIndex < startIdx) {
+        // need to find first nt in subsequence
+        subsequence.FirstNonTerminalIndex = -1;
+        for (int i = 0; subsequence.FirstNonTerminalIndex == -1 && i < len; i++) {
+          if (subsequence[i] >= 'A' && subsequence[i] <= 'Z') subsequence.FirstNonTerminalIndex = i;
+        }
+      } else if (FirstNonTerminalIndex >= startIdx && FirstNonTerminalIndex < startIdx + len) {
+        subsequence.FirstNonTerminalIndex = FirstNonTerminalIndex - startIdx;
+      } else {
+        Debug.Assert(FirstNonTerminalIndex >= startIdx + len);
+        subsequence.FirstNonTerminalIndex = -1;
+      }
+      return subsequence;
+    }
+
+    public override bool Equals(object obj) {
+      var other = obj as ReadonlySequence;
+      if (other == null) return false;
+      if (other.Length != this.Length) return false;
+      for (int i = 0; i < Length; i++)
+        if (other[i] != this[i]) return false;
+
+      // length and all symbols are the same
+      return true;
+    }
+
+    public override int GetHashCode() {
+      int h = 31 * Length;
+      for (int i = 0; i < Length; i++) {
+        h += 31 * (byte)this[i];
+      }
+      return h;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/SentenceSetStatistics.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/SentenceSetStatistics.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/SentenceSetStatistics.cs	(revision 12290)
@@ -0,0 +1,71 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Common;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  public class SentenceSetStatistics {
+    public int NumberOfSentences { get; private set; }
+    public string BestSentence { get; private set; }
+    public string FirstSentence { get; private set; }
+    public string LastSentence { get; private set; }
+    public double BestSentenceQuality { get; private set; }
+    public double BestSentenceIndex { get; private set; }
+    public double FirstSentenceQuality { get; private set; }
+    public double LastSentenceQuality { get; private set; }
+    public double AverageQuality { get { return sumQualities / NumberOfSentences; } }
+    // public double MedianQuality { get; private set; }
+    private double sumQualities;
+
+    private readonly double bestKnownQuality;
+    public SentenceSetStatistics(double bestKnownQuality = 1.0) {
+      this.bestKnownQuality = bestKnownQuality;
+      BestSentenceQuality = double.NegativeInfinity;
+      BestSentence = string.Empty;
+      FirstSentence = string.Empty;
+      LastSentence = string.Empty;
+    }
+
+    public void AddSentence(string sentence, double quality) {
+      sumQualities += quality;
+      NumberOfSentences++;
+
+      if (NumberOfSentences == 1) {
+        FirstSentence = sentence;
+        FirstSentenceQuality = quality;
+      }
+
+      if (quality > BestSentenceQuality) {
+        BestSentence = sentence;
+        BestSentenceQuality = quality;
+        BestSentenceIndex = NumberOfSentences;
+      }
+
+      LastSentence = sentence;
+      LastSentenceQuality = quality;
+    }
+
+    public override string ToString() {
+      return
+        string.Format("Sentences: {0,10} avg.-quality {1,7:F5} best {2,7:F5} {3,2} {4,10} {5,30} first {6,7:F5} {7,20} last {8,7:F5} {9,20}",
+      NumberOfSentences, AverageQuality,
+      BestSentenceQuality, DoubleExtensions.IsAlmost(BestSentenceQuality, bestKnownQuality) ? 1.0 : 0.0,
+      BestSentenceIndex, TrimToSize(BestSentence, 30),
+      FirstSentenceQuality, TrimToSize(FirstSentence, 20),
+      LastSentenceQuality, TrimToSize(LastSentence, 20)
+     );
+    }
+
+    private string TrimToSize(string s, int len) {
+      if (s.Length < len) return s;
+      else {
+        var sb = new StringBuilder(len);
+        sb.Append(s.Substring(0, len - 3));
+        sb.Append("...");
+        return sb.ToString();
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Sequence.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Sequence.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/HeuristicLab.Problems.GrammaticalOptimization/Sequence.cs	(revision 12290)
@@ -0,0 +1,160 @@
+﻿using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization {
+  // represents a sequence of symbols (non-terminal and terminal symbols)
+  // a sequence consisting only of terminal symbols can be a sentence of a language
+  // the class supports in-place manipulation of the sequence symbols (replace NT with another sequence)
+  // sequences provide efficient support left-canonical derivation by storing the index of the first non-terminal symbol
+  // maximal length of sequences is limited to 1000 symbols
+
+  // for symbols the same assumptions for the implementation of grammars apply
+  // - non-terminal symbols must be characters in the range [A..Z]
+  // - terminal symbols can be almost all other characters
+  public class Sequence : IEnumerable<char> {
+    private const int maxIdx = 200;
+
+    private int len;
+    private int idxOfFirstNt;
+    protected char[] symbols;
+
+    public virtual char this[int idx] {
+      get { return symbols[idx]; }
+      set { throw new NotSupportedException(); }
+    }
+
+    public int Length {
+      get { return len; }
+      protected set { len = value; }
+    }
+
+    public bool IsTerminal {
+      get { return idxOfFirstNt == -1; }
+    }
+
+    public int FirstNonTerminalIndex {
+      get { return idxOfFirstNt; }
+      protected set { idxOfFirstNt = value; }
+    }
+
+    public char FirstNonTerminal {
+      get { return this[idxOfFirstNt]; }
+    }
+
+    private Sequence(int maxLength) {
+      this.symbols = new char[maxLength];
+    }
+
+    // create a sequence from a character
+    public Sequence(char ch)
+      : this(ch, maxIdx + 1) {
+    }
+
+    protected Sequence(char ch, int maxLength)
+      : this(maxLength) {
+      this.len = 1;
+      symbols[0] = ch;
+
+      if (ch >= 'A' && ch <= 'Z') idxOfFirstNt = 0;
+      else idxOfFirstNt = -1;
+    }
+
+    // create a sequence from a string
+    public Sequence(string s) : this(s, maxIdx + 1) { }
+    protected Sequence(string s, int maxLength)
+      : this(maxLength) {
+      if (string.IsNullOrEmpty(s)) throw new ArgumentException();
+      if (s.Length > (maxIdx + 1)) throw new ArgumentException();
+      this.len = s.Length;
+      this.idxOfFirstNt = -1;
+
+      for (int i = 0; i < len; i++) {
+        symbols[i] = s[i];
+        if (idxOfFirstNt == -1 && symbols[i] >= 'A' && symbols[i] <= 'Z') {
+          idxOfFirstNt = i;
+        }
+      }
+    }
+
+    // cloning ctor
+    public Sequence(Sequence original) : this(original, maxIdx + 1) { }
+    protected Sequence(Sequence original, int maxLength)
+      : this(maxLength) {
+      this.len = original.len;
+      Array.Copy(original.symbols, this.symbols, len);
+      this.idxOfFirstNt = original.idxOfFirstNt;
+    }
+
+    // empty constructor does not allocate the symbol array
+    protected Sequence() { }
+
+    public virtual void ReplaceAt(int position, int len, Sequence replacement) {
+      if (replacement == null) throw new ArgumentNullException();
+      if (len <= 0) throw new ArgumentException();
+      if (position + len >= maxIdx) throw new ArgumentException();
+      if (Length - len + replacement.Length > (maxIdx + 1)) throw new ArgumentException();
+      var lenDelta = replacement.Length - len;
+      var remainingPartLen = Length - position - len;
+      var startIdxOfRemainingPart = position + len + lenDelta;
+      Array.Copy(symbols, position + len, symbols, startIdxOfRemainingPart, remainingPartLen);
+      Array.Copy(replacement.symbols, 0, symbols, position, replacement.Length);
+
+      this.len = Length + lenDelta;
+      // when the first part contains an NT then it's not necessary to update the index
+      if (idxOfFirstNt >= 0 && idxOfFirstNt < position) return;
+      // if the replacement contains an NT then we can directly calculate the idx of that NT in the new sequence
+      if (replacement.idxOfFirstNt >= 0) {
+        this.idxOfFirstNt = position + replacement.idxOfFirstNt;
+      } else {
+        // otherwise we must find the first NT in the remaining part
+        idxOfFirstNt = -1;
+        for (int i = startIdxOfRemainingPart; idxOfFirstNt == -1 && i < Length; i++) {
+          if (this[i] >= 'A' && this[i] <= 'Z') idxOfFirstNt = i;
+        }
+      }
+    }
+
+    public virtual IEnumerator<char> GetEnumerator() {
+      return symbols.AsEnumerable().Take(len).GetEnumerator();
+    }
+
+    public override string ToString() {
+      var sb = new StringBuilder(len);
+      sb.Append(symbols, 0, len);
+      return sb.ToString();
+    }
+
+    IEnumerator IEnumerable.GetEnumerator() {
+      return GetEnumerator();
+    }
+
+    public Sequence Subsequence(int startIdx, int len) {
+      if (startIdx < 0 || len < 0) throw new ArgumentException();
+      if (startIdx >= this.len) throw new ArgumentException();
+      if (startIdx + len > this.len) throw new ArgumentException();
+      var subsequence = new Sequence(maxIdx + 1) { len = len };
+
+      Array.Copy(this.symbols, startIdx, subsequence.symbols, 0, len);
+      if (idxOfFirstNt < 0) {
+        subsequence.idxOfFirstNt = -1;
+      } else if (idxOfFirstNt < startIdx) {
+        // need to find first nt in subsequence
+        subsequence.idxOfFirstNt = -1;
+        for (int i = 0; subsequence.idxOfFirstNt == -1 && i < len; i++) {
+          if (subsequence[i] >= 'A' && subsequence[i] <= 'Z') subsequence.idxOfFirstNt = i;
+        }
+      } else if (idxOfFirstNt >= startIdx && idxOfFirstNt < startIdx + len) {
+        subsequence.idxOfFirstNt = idxOfFirstNt - startIdx;
+      } else {
+        Debug.Assert(idxOfFirstNt >= startIdx + len);
+        subsequence.idxOfFirstNt = -1;
+      }
+      return subsequence;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Main.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Main.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Main.csproj	(revision 12290)
@@ -0,0 +1,78 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{524CBD70-6F99-46AF-AD00-A0BC9FD1175B}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>Main</RootNamespace>
+    <AssemblyName>Main</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.Bandits\HeuristicLab.Algorithms.Bandits.csproj">
+      <Project>{24408F7D-EE0F-4886-A08B-EC324D662E47}</Project>
+      <Name>HeuristicLab.Algorithms.Bandits</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.GrammaticalOptimization\HeuristicLab.Algorithms.GrammaticalOptimization.csproj">
+      <Project>{eea07488-1a51-412a-a52c-53b754a628b3}</Project>
+      <Name>HeuristicLab.Algorithms.GrammaticalOptimization</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.MonteCarloTreeSearch\HeuristicLab.Algorithms.MonteCarloTreeSearch.csproj">
+      <Project>{2c115235-8fa9-4f7f-b3a0-a0144f8a35ca}</Project>
+      <Name>HeuristicLab.Algorithms.MonteCarloTreeSearch</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization.SymbReg\HeuristicLab.Problems.GrammaticalOptimization.SymbReg.csproj">
+      <Project>{17A7A380-86CE-482D-8D22-CBD70CC97F0D}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization.SymbReg</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.csproj">
+      <Project>{cb9dccf6-667e-4a13-b82d-dbd6b45a045e}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="app.config" />
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Program.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Program.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Program.cs	(revision 12290)
@@ -0,0 +1,95 @@
+using System;
+using System.Diagnostics;
+using System.Globalization;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Algorithms.MonteCarloTreeSearch;
+using HeuristicLab.Algorithms.MonteCarloTreeSearch.Simulation;
+using HeuristicLab.Problems.GrammaticalOptimization;
+
+// NOTES: gkronber
+// TODO: feature extraction for full symbolic expressions and experiment for all benchmark problems
+// TODO: why does GaussianThompsonSampling work so well with MCTS for the artificial ant problem?
+// TODO: research thompson sampling for max bandit?
+// TODO: verify TA implementation using example from the original paper      
+// TODO: implement thompson sampling for gaussian mixture models
+// TODO: gleichzeitige modellierung von transformierter zielvariable (y, 1/y, log(y), exp(y), sqrt(y), ...)
+// TODO: vergleich bei complete-randomly mÃ¶glichst kurze sÃ€tze generieren vs. einfach zufÃ€llig alternativen wÃ€hlen
+// TODO: reward discounting (fÃŒr verÃ€nderliche reward distributions ÃŒber zeit). speziellen unit-test dafÃŒr erstellen
+// TODO: constant optimization 
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+
+
+namespace Main {
+  class Program {
+    static void Main(string[] args) {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      RunDemo();
+    }
+
+
+    private static void RunDemo() {
+
+      for (int i = 0; i < 100; i++) {
+        int maxIterations = 2000000;
+        int iterations = 0;
+
+        var globalStatistics = new SentenceSetStatistics();
+        var random = new Random();
+
+        var problem = new SymbolicRegressionPoly10Problem();
+        //var problem = new SantaFeAntProblem();              
+        //var problem = new RoyalPairProblem(25);
+        //var problem = new FindPhrasesProblem(random, 10, 5, 3, 5, 5, 1.0, 0.9, true);
+        //var problem = new PrimePolynomialProblem();
+        //var problem = new SymbolicRegressionProblem(random,
+        //  //@"C:\reps\HeuristicLab\branches\HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.SymbReg\nht-train.csv",
+        //  @"C:\reps\fhooe-new\research\Datasets\Benchmark\kommenda-1.csv",
+        //  1.0,
+        //  true);
+        // //var problem = new PrimePolynomialProblem();
+        // var alg = new SequentialSearch(problem, 25, random, 0,
+        //   new HeuristicLab.Algorithms.Bandits.GrammarPolicies.GenericGrammarPolicy(problem, new UCB1TunedPolicy()));
+        var alg = new SequentialSearch(problem, 25, random, 0,
+          new GenericPolicy(problem, new HeuristicLab.Algorithms.Bandits.BanditPolicies.EpsGreedyPolicy(0.1)));
+        //var alg = new MonteCarloTreeSearch(problem, 23, random, new UCB1Policy(), new RandomSimulation(problem, random, 30));
+
+
+        alg.FoundNewBestSolution += (sentence, quality) => {
+          //Console.WriteLine("{0}", globalStatistics);
+        };
+
+        alg.SolutionEvaluated += (sentence, quality) => {
+          iterations++;
+          globalStatistics.AddSentence(sentence, quality);
+
+          // comment this if you don't want to see solver statistics
+          if (iterations % 100 == 0) {
+            if (iterations % 1000 == 0) Console.Clear();
+            Console.SetCursorPosition(0, 0);
+            alg.PrintStats();
+          }
+
+          // uncomment this if you want to collect statistics of the generated sentences
+          //if (iterations % 100 == 0) {
+          //  Console.WriteLine("{0}", globalStatistics);
+          //}
+        };
+
+        var sw = new Stopwatch();
+        sw.Start();
+        alg.Run(maxIterations);
+        sw.Stop();
+
+        Console.WriteLine(globalStatistics);
+
+        Console.WriteLine("{0:F2} sec {1,10:F1} sols/sec {2,10:F1} ns/sol",
+          sw.Elapsed.TotalSeconds,
+          maxIterations / (double)sw.Elapsed.TotalSeconds,
+          (double)sw.ElapsedMilliseconds * 1000 / maxIterations);
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("Main")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("Main")]
+[assembly: AssemblyCopyright("Copyright © Microsoft 2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("6c9d053b-ed5d-4285-868a-1b6ea7587812")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/app.config
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/app.config	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Main/app.config	(revision 12290)
@@ -0,0 +1,3 @@
+<?xml version="1.0"?>
+<configuration>
+<startup><supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5"/></startup></configuration>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/Properties/AssemblyInfo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/Properties/AssemblyInfo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/Properties/AssemblyInfo.cs	(revision 12290)
@@ -0,0 +1,36 @@
+﻿using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("HeuristicLab.Problems.GrammaticalOptimization.Test")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("HeuristicLab.Problems.GrammaticalOptimization.Test")]
+[assembly: AssemblyCopyright("Copyright © Microsoft 2014")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("1f7e7cac-1ef2-43dc-b5da-5547058c9309")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunBaselineExperiments.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunBaselineExperiments.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunBaselineExperiments.cs	(revision 12290)
@@ -0,0 +1,312 @@
+﻿using System;
+using System.Collections;
+using System.Collections.Generic;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.GeneticProgramming;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using RandomPolicy = HeuristicLab.Algorithms.Bandits.BanditPolicies.RandomPolicy;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+
+
+  [TestClass]
+  public class RunBaselineExperiments {
+    private readonly static int randSeed = 31415;
+
+    internal class Configuration {
+      public ISymbolicExpressionTreeProblem Problem;
+      public int MaxSize;
+      public int RandSeed;
+
+      public override string ToString() {
+        return string.Format("{0} {1} {2}", RandSeed, Problem, MaxSize);
+      }
+    }
+
+    #region permutation
+    [TestMethod]
+    public void RunPureRandomSearchPermutationProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
+      };
+
+      var maxSizes = new int[] { 32 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunRandomSearchNoResamplingPermutationProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
+      };
+
+      var maxSizes = new int[] { 32 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    #endregion
+
+    #region royalpair
+    [TestMethod]
+    public void RunPureRandomSearchRoyalPairProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunRandomSearchNoResamplingRoyalPairProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    #endregion
+
+    #region royalsymbol
+    [TestMethod]
+    public void RunPureRandomSearchRoyalSymbolProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunRandomSearchNoResamplingRoyalSymbolProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    #endregion
+
+    #region findphrases
+    [TestMethod]
+    public void RunPureRandomSearchFindPhrasesProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
+      };
+
+      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunRandomSearchNoResamplingFindPhrasesProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
+      };
+
+      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+
+    #endregion
+
+    #region artificial ant
+    [TestMethod]
+    public void RunPureRandomSearchArtificialAntProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
+      int nReps = 20;
+      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunRandomSearchNoResamplingArtificialAntProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
+      int nReps = 20;
+      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    #endregion
+
+    #region symb-reg-poly-10
+    [TestMethod]
+    public void RunPureRandomSearchPoly10Problem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
+      int nReps = 20;
+      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunPureRandomSearchForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunRandomSearchNoResamplingPoly10Problem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
+      int nReps = 20;
+      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunRandomSearchNoResamplingForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+    #endregion
+
+    #region helpers
+    private IEnumerable<Configuration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
+      int nReps,
+      IEnumerable<int> maxSizes
+      ) {
+      var seedRand = new Random(randSeed);
+      // the problem seed is the same for all configuratons 
+      // this guarantees that we solve the _same_ problem each time
+      // with different solvers and multiple repetitions
+      var problemSeed = randSeed;
+      for (int i = 0; i < nReps; i++) {
+        // in each repetition use the same random seed for all solver configuratons
+        // do nReps with different seeds for each configuration
+        var solverSeed = seedRand.Next();
+        foreach (var maxSize in maxSizes) {
+          yield return new Configuration {
+            MaxSize = maxSize,
+            Problem = problemFactory(problemSeed),
+            RandSeed = solverSeed
+          };
+        }
+      }
+    }
+
+    private static void RunPureRandomSearchForProblem(
+      int randSeed,
+      ISymbolicExpressionTreeProblem problem,
+      int maxIters,
+      int maxSize
+      ) {
+      var solver = new RandomSearch(problem, new Random(randSeed), maxSize);
+      var problemName = problem.GetType().Name;
+      var bestKnownQuality = problem.BestKnownQuality(maxSize);
+      RunSolver(solver, problemName, bestKnownQuality, maxIters, maxSize);
+    }
+
+    private static void RunRandomSearchNoResamplingForProblem(
+  int randSeed,
+  ISymbolicExpressionTreeProblem problem,
+  int maxIters,
+  int maxSize
+  ) {
+      var solver = new SequentialSearch(problem, maxSize, new Random(randSeed), 0,
+        new GenericGrammarPolicy(problem, new RandomPolicy(), false));
+
+      var problemName = problem.GetType().Name;
+      var bestKnownQuality = problem.BestKnownQuality(maxSize);
+      RunSolver(solver, problemName, bestKnownQuality, maxIters, maxSize);
+    }
+
+    private static void RunSolver(ISolver solver, string problemName, double bestKnownQuality, int maxIters, int maxSize) {
+      int iterations = 0;
+      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
+      var gpName = solver.GetType().Name;
+      solver.SolutionEvaluated += (sentence, quality) => {
+        iterations++;
+        globalStatistics.AddSentence(sentence, quality);
+
+        if (iterations % 1000 == 0) {
+          Console.WriteLine("\"{0,25}\" {1} \"{2,25}\" {3}", gpName, maxSize, problemName, globalStatistics);
+        }
+      };
+
+      solver.Run(maxIters);
+    }
+    #endregion
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunDemo.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunDemo.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunDemo.cs	(revision 12290)
@@ -0,0 +1,163 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.Bandits.Models;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using RandomPolicy = HeuristicLab.Algorithms.Bandits.BanditPolicies.RandomPolicy;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  [TestClass]
+  public class RunDemo {
+    [TestMethod]
+    public void RunGridTest() {
+      int maxIterations = 20000; // for poly-10 with 50000 evaluations no successful try with hl yet
+      //var globalRandom = new Random(31415);
+      var localRandSeed = new Random().Next();
+      var reps = 20;
+
+      var policyFactories = new Func<IBanditPolicy>[]
+        {
+         () => new RandomPolicy(), 
+          () => new ActiveLearningPolicy(),  
+         // () => new EpsGreedyPolicy(0.01, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.05, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.1, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.2, (aInfo)=> aInfo.MaxReward, "max"), 
+         //() => new GaussianThompsonSamplingPolicy(), 
+         () => new GaussianThompsonSamplingPolicy(true), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1)), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1, 1)), 
+         //() => new BernoulliThompsonSamplingPolicy(),
+         () => new GenericThompsonSamplingPolicy(new BernoulliModel(1, 1)), 
+         () => new EpsGreedyPolicy(0.01), 
+         () => new EpsGreedyPolicy(0.05), 
+         () => new EpsGreedyPolicy(0.1), 
+         () => new EpsGreedyPolicy(0.2), 
+         () => new EpsGreedyPolicy(0.5), 
+         () => new UCTPolicy(0.01),
+         () => new UCTPolicy(0.05),
+         () => new UCTPolicy(0.1),
+         () => new UCTPolicy(0.5),
+         () => new UCTPolicy(1),
+         () => new UCTPolicy(2),
+         () => new UCTPolicy( 5),
+         () => new UCTPolicy( 10),
+         () => new ModifiedUCTPolicy(0.01),
+         () => new ModifiedUCTPolicy(0.05),
+         () => new ModifiedUCTPolicy(0.1),
+         () => new ModifiedUCTPolicy(0.5),
+         () => new ModifiedUCTPolicy(1),
+         () => new ModifiedUCTPolicy(2),
+         () => new ModifiedUCTPolicy( 5),
+         () => new ModifiedUCTPolicy( 10),
+         () => new UCB1Policy(), 
+         () => new UCB1TunedPolicy(), 
+         () => new UCBNormalPolicy(), 
+         () => new BoltzmannExplorationPolicy(1),
+         () => new BoltzmannExplorationPolicy(10),
+         () => new BoltzmannExplorationPolicy(20),
+         () => new BoltzmannExplorationPolicy(100),
+         () => new BoltzmannExplorationPolicy(200),
+         () => new BoltzmannExplorationPolicy(500),
+          () => new ChernoffIntervalEstimationPolicy( 0.01), 
+          () => new ChernoffIntervalEstimationPolicy( 0.05),
+          () => new ChernoffIntervalEstimationPolicy( 0.1),
+          () => new ChernoffIntervalEstimationPolicy( 0.2),
+         () => new ThresholdAscentPolicy(5, 0.01), 
+         () => new ThresholdAscentPolicy(5, 0.05),
+         () => new ThresholdAscentPolicy(5, 0.1),
+         () => new ThresholdAscentPolicy(5, 0.2),
+         () => new ThresholdAscentPolicy(10, 0.01), 
+         () => new ThresholdAscentPolicy(10, 0.05),
+         () => new ThresholdAscentPolicy(10, 0.1),
+         () => new ThresholdAscentPolicy(10, 0.2),
+         () => new ThresholdAscentPolicy(50, 0.01), 
+         () => new ThresholdAscentPolicy(50, 0.05),
+         () => new ThresholdAscentPolicy(50, 0.1),
+         () => new ThresholdAscentPolicy(50, 0.2),
+         () => new ThresholdAscentPolicy(100, 0.01), 
+         () => new ThresholdAscentPolicy(100, 0.05),
+         () => new ThresholdAscentPolicy(100, 0.1),
+         () => new ThresholdAscentPolicy(100, 0.2),
+         () => new ThresholdAscentPolicy(500, 0.01), 
+         () => new ThresholdAscentPolicy(500, 0.05),
+         () => new ThresholdAscentPolicy(500, 0.1),
+         () => new ThresholdAscentPolicy(500, 0.2),
+         () => new ThresholdAscentPolicy(5000, 0.01), 
+         () => new ThresholdAscentPolicy(10000, 0.01), 
+        };
+
+      var instanceFactories = new Func<Random, Tuple<IProblem, int>>[]
+      {
+        //(rand) => Tuple.Create((IProblem)new SantaFeAntProblem(), 17),
+        //(rand) => Tuple.Create((IProblem)new FindPhrasesProblem(rand, 10, numPhrases:5, phraseLen:3, numOptimalPhrases:5, numDecoyPhrases:0, correctReward:1, decoyReward:0, phrasesAsSets:false ), 15),
+        //(rand) => Tuple.Create((IProblem)new FindPhrasesProblem(rand, 10, numPhrases:5, phraseLen:3, numOptimalPhrases:5, numDecoyPhrases:0, correctReward:1, decoyReward:0, phrasesAsSets:true ), 15),
+        //(rand) => Tuple.Create((IProblem)new FindPhrasesProblem(rand, 10, numPhrases:5, phraseLen:3, numOptimalPhrases:5, numDecoyPhrases:200, correctReward:1, decoyReward:0.5, phrasesAsSets:false), 15),
+        //(rand) => Tuple.Create((IProblem)new FindPhrasesProblem(rand, 10, numPhrases:5, phraseLen:3, numOptimalPhrases:5, numDecoyPhrases:200, correctReward:1, decoyReward:0.5, phrasesAsSets:true), 15),
+        //(rand) => Tuple.Create((IProblem)new SymbolicRegressionPoly10Problem(), 23)
+        (rand) => Tuple.Create((IProblem)new SantaFeAntProblem(), 17)
+      };
+
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var useCanonical in new bool[] { true /*, false */ }) {
+          foreach (var randomTries in new int[] { 0 /*, 1, 10 /*, /* 5, 100 /*, 500, 1000 */}) {
+            foreach (var policyFactory in policyFactories) {
+              var myRandomTries = randomTries;
+              var localRand = new Random(localRandSeed);
+              var options = new ParallelOptions();
+              options.MaxDegreeOfParallelism = 1;
+              Parallel.For(0, reps, options, (i) => {
+                Random myLocalRand;
+                lock (localRand)
+                  myLocalRand = new Random(localRand.Next());
+
+                int iterations = 0;
+                var globalStatistics = new SentenceSetStatistics();
+
+                // var problem = new SymbolicRegressionPoly10Problem();
+                // var problem = new SantaFeAntProblem();
+                //var problem = new PalindromeProblem();
+                //var problem = new HardPalindromeProblem();
+                //var problem = new RoyalPairProblem();
+                //var problem = new EvenParityProblem();
+                // var alg = new MctsSampler(problem.Item1, problem.Item2, myLocalRand, myRandomTries, policy()); 
+                var instance = instanceFactory(myLocalRand);
+                var problem = instance.Item1;
+                var maxLen = instance.Item2;
+                var alg = new SequentialSearch(problem, maxLen, myLocalRand, myRandomTries,
+                  new GenericGrammarPolicy(problem, policyFactory(), useCanonical));
+                // var alg = new SequentialSearch(problem, maxLen, myLocalRand,
+                //   myRandomTries,
+                //   new GenericFunctionApproximationGrammarPolicy(problem,
+                //     useCanonical));
+                //var alg = new ExhaustiveBreadthFirstSearch(problem, 25);
+                //var alg = new AlternativesContextSampler(problem, 25);
+
+                alg.SolutionEvaluated += (sentence, quality) => {
+                  iterations++;
+                  globalStatistics.AddSentence(sentence, quality);
+                  if (iterations % 1000 == 0) {
+                    Console.WriteLine("{0,3} {1,5} \"{2,25}\" {3} {4} {5}", i, myRandomTries, policyFactory(), useCanonical, problem.ToString(), globalStatistics);
+                  }
+                };
+                alg.FoundNewBestSolution += (sentence, quality) => {
+                  //Console.WriteLine("{0,5} {1,25} {2} {3}",
+                  //  myRandomTries, policyFactory(), useCanonical,
+                  //  globalStatistics);
+                };
+
+                alg.Run(maxIterations);
+              });
+            }
+          }
+        }
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunGpExperiments.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunGpExperiments.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunGpExperiments.cs	(revision 12290)
@@ -0,0 +1,328 @@
+﻿using System;
+using System.Collections;
+using System.Collections.Generic;
+using HeuristicLab.Algorithms.GeneticProgramming;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+
+
+  [TestClass]
+  public class RunGpExperiments {
+    private readonly static int[] popSizes = new int[] { 100, 250, 500, 1000, 2500, 5000, 10000 };
+    private readonly static double[] mutationRates = new double[] { 0.15 };
+    private readonly static int randSeed = 31415;
+
+    internal class GPConfiguration {
+      public ISymbolicExpressionTreeProblem Problem;
+      public int PopSize;
+      public int MaxSize;
+      public int RandSeed;
+      public double MutationRate;
+
+      public override string ToString() {
+        return string.Format("{0} {1} {2} {3} {4}", RandSeed, Problem, PopSize, MaxSize, MutationRate);
+      }
+    }
+
+    #region permutation
+    [TestMethod]
+    public void RunStandardGPPermutationProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
+      };
+
+      var maxSizes = new int[] { 32 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunOffspringSelectionGPPermutationProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new PermutationProblem(), 
+      };
+
+      var maxSizes = new int[] { 32 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    #endregion
+
+    #region royalpair
+    [TestMethod]
+    public void RunStandardGPRoyalPairProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunOffspringSelectionGPRoyalPairProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalPairProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    #endregion
+
+    #region royalsymbol
+    [TestMethod]
+    public void RunStandardGPRoyalSymbolProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunOffspringSelectionGPRoyalSymbolProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new RoyalSymbolProblem(), 
+      };
+
+      var maxSizes = new int[] { 32, 64, 128, 256 };
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    #endregion
+
+    #region findphrases
+    [TestMethod]
+    public void RunStandardGPFindPhrasesProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
+      };
+
+      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunOffspringSelectionGPFindPhrasesProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 0, 1, 0, false), 
+        (randSeed) => (ISymbolicExpressionTreeProblem) new FindPhrasesProblem(new Random(randSeed), 20, 5, 3, 5, 50, 1, 0.8, false), 
+      };
+
+      var maxSizes = new int[] { 15 * 3 }; // * 3 for non-terminals
+      int nReps = 20;
+      int maxIterations = 50000;
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    #endregion
+
+    #region artificial ant
+    [TestMethod]
+    public void RunStandardGPArtificialAntProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
+      int nReps = 20;
+      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunOffspringSelectionGPArtificialAntProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 17
+      int nReps = 20;
+      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    #endregion
+
+    #region symb-reg-poly-10
+    [TestMethod]
+    public void RunStandardGPPoly10Problem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
+      int nReps = 20;
+      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunStandardGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    [TestMethod]
+    public void RunOffspringSelectionGPPoly10Problem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var maxSizes = new int[] { 30, 50, 100 }; // size of sequential representation is 23
+      int nReps = 20;
+      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, popSizes, maxSizes, mutationRates)) {
+          RunOffspringSelectionGpForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.PopSize, conf.MaxSize, conf.MutationRate);
+        }
+      }
+    }
+    #endregion
+
+    #region helpers
+    private IEnumerable<GPConfiguration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
+      int nReps,
+      IEnumerable<int> popSizes,
+      IEnumerable<int> maxSizes,
+      IEnumerable<double> mutationRates) {
+      var seedRand = new Random(randSeed);
+      // the problem seed is the same for all configuratons 
+      // this guarantees that we solve the _same_ problem each time
+      // with different solvers and multiple repetitions
+      var problemSeed = randSeed;
+      for (int i = 0; i < nReps; i++) {
+        // in each repetition use the same random seed for all solver configuratons
+        // do nReps with different seeds for each configuration
+        var solverSeed = seedRand.Next();
+        foreach (var popSize in popSizes) {
+          foreach (var mutRate in mutationRates) {
+            foreach (var maxSize in maxSizes) {
+              yield return new GPConfiguration {
+                MaxSize = maxSize,
+                MutationRate = mutRate,
+                PopSize = popSize,
+                Problem = problemFactory(problemSeed),
+                RandSeed = solverSeed
+              };
+            }
+          }
+        }
+      }
+    }
+
+    private static void RunStandardGpForProblem(
+      int randSeed,
+      ISymbolicExpressionTreeProblem problem,
+      int maxIters,
+      int popSize,
+      int maxSize,
+      double mutationRate
+      ) {
+      var gp = new StandardGP(problem, new Random(randSeed), false);
+      var problemName = problem.GetType().Name;
+      var bestKnownQuality = problem.BestKnownQuality(maxSize);
+      RunGP(gp, problemName, bestKnownQuality, maxIters, popSize, mutationRate, maxSize);
+    }
+
+    private static void RunOffspringSelectionGpForProblem(
+  int randSeed,
+  ISymbolicExpressionTreeProblem problem,
+  int maxIters,
+  int popSize,
+  int maxSize,
+  double mutationRate
+  ) {
+      var gp = new OffspringSelectionGP(problem, new Random(randSeed), false);
+      var problemName = problem.GetType().Name;
+      var bestKnownQuality = problem.BestKnownQuality(maxSize);
+      RunGP(gp, problemName, bestKnownQuality, maxIters, popSize, mutationRate, maxSize);
+    }
+
+
+
+
+    private static void RunGP(IGPSolver gp, string problemName, double bestKnownQuality, int maxIters, int popSize, double mutationRate, int maxSize) {
+      int iterations = 0;
+      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
+      var gpName = gp.GetType().Name;
+      gp.SolutionEvaluated += (sentence, quality) => {
+        iterations++;
+        globalStatistics.AddSentence(sentence, quality);
+
+        if (iterations % 1000 == 0) {
+          Console.WriteLine("\"{0,25}\" {1} {2:N2} {3} \"{4,25}\" {5}", gpName, popSize, mutationRate, maxSize, problemName, globalStatistics);
+        }
+      };
+
+      gp.PopulationSize = popSize;
+      gp.MutationRate = mutationRate;
+      gp.MaxSolutionSize = maxSize + 2;
+      gp.MaxSolutionDepth = maxSize + 2;
+
+      gp.Run(maxIters);
+    }
+    #endregion
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunMctsExperiments.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunMctsExperiments.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunMctsExperiments.cs	(revision 12290)
@@ -0,0 +1,123 @@
+﻿using System;
+using System.Collections;
+using System.Collections.Generic;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.GeneticProgramming;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+
+  [TestClass]
+  public class RunMctsExperiments {
+    private readonly static int randSeed = 31415;
+
+    internal class Configuration {
+      public ISymbolicExpressionTreeProblem Problem;
+      public int MaxSize;
+      public int RandSeed;
+
+      public override string ToString() {
+        return string.Format("{0} {1} {2}", RandSeed, Problem, MaxSize);
+      }
+    }
+
+
+    #region artificial ant
+    [TestMethod]
+    public void RunSeqSolvFuncApproxArtificialAntProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
+      };
+
+      var maxSizes = new int[] { 17 }; // size of sequential representation is 17
+      int nReps = 30;
+      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunMctsForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+
+    #endregion
+
+    #region symb-reg-poly-10
+    [TestMethod]
+    public void RunSeqSolvFuncApproxPoly10Problem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var maxSizes = new int[] { 23 }; // size of sequential representation is 23
+      int nReps = 30;
+      int maxIterations = 200000; // sequentialsearch should find the optimum within 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunMctsForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+
+    #endregion
+
+    #region helpers
+    private IEnumerable<Configuration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
+      int nReps,
+      IEnumerable<int> maxSizes
+      ) {
+      var seedRand = new Random(randSeed);
+      // the problem seed is the same for all configuratons 
+      // this guarantees that we solve the _same_ problem each time
+      // with different solvers and multiple repetitions
+      var problemSeed = randSeed;
+      for (int i = 0; i < nReps; i++) {
+        // in each repetition use the same random seed for all solver configuratons
+        // do nReps with different seeds for each configuration
+        var solverSeed = seedRand.Next();
+        foreach (var maxSize in maxSizes) {
+          yield return new Configuration {
+            MaxSize = maxSize,
+            Problem = problemFactory(problemSeed),
+            RandSeed = solverSeed
+          };
+        }
+      }
+    }
+
+    private static void RunMctsForProblem(
+      int randSeed,
+      IProblem problem,
+      int maxIters,
+      int maxSize
+      ) {
+      var solver = new SequentialSearch(problem, maxSize, new Random(randSeed), 0,
+        new GenericGrammarPolicy(problem, new UCB1Policy(), false));
+      var problemName = problem.GetType().Name;
+      var bestKnownQuality = problem.BestKnownQuality(maxSize);
+      RunSolver(solver, problemName, bestKnownQuality, maxIters, maxSize);
+    }
+
+    private static void RunSolver(ISolver solver, string problemName, double bestKnownQuality, int maxIters, int maxSize) {
+      int iterations = 0;
+      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
+      var gpName = solver.GetType().Name;
+      solver.SolutionEvaluated += (sentence, quality) => {
+        iterations++;
+        globalStatistics.AddSentence(sentence, quality);
+
+        if (iterations % 1000 == 0) {
+          Console.WriteLine("\"{0,25}\" {1} \"{2,25}\" {3}", gpName, maxSize, problemName, globalStatistics);
+        }
+      };
+
+      solver.Run(maxIters);
+    }
+    #endregion
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunSequentialSolverFuncApproxExperiments.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunSequentialSolverFuncApproxExperiments.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/RunSequentialSolverFuncApproxExperiments.cs	(revision 12290)
@@ -0,0 +1,123 @@
+﻿using System;
+using System.Collections;
+using System.Collections.Generic;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.GeneticProgramming;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  // only supported for Artificial Ant and Symbolic Regression up to now
+
+  [TestClass]
+  public class RunSequentialSolverFuncApproximationExperiments {
+    private readonly static int randSeed = 31415;
+
+    internal class Configuration {
+      public ISymbolicExpressionTreeProblem Problem;
+      public int MaxSize;
+      public int RandSeed;
+
+      public override string ToString() {
+        return string.Format("{0} {1} {2}", RandSeed, Problem, MaxSize);
+      }
+    }
+
+
+    #region artificial ant
+    [TestMethod]
+    public void RunSeqSolvFuncApproxArtificialAntProblem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SantaFeAntProblem(), 
+      };
+
+      var maxSizes = new int[] { 17 }; // size of sequential representation is 17
+      int nReps = 20;
+      int maxIterations = 100000; // randomsearch finds the optimum almost always for 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunSeqSolvFuncApproxForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+
+    #endregion
+
+    #region symb-reg-poly-10
+    [TestMethod]
+    public void RunSeqSolvFuncApproxPoly10Problem() {
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem) new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var maxSizes = new int[] { 23 }; // size of sequential representation is 23
+      int nReps = 20;
+      int maxIterations = 100000; // sequentialsearch should find the optimum within 100000 evals
+      foreach (var instanceFactory in instanceFactories) {
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          RunSeqSolvFuncApproxForProblem(conf.RandSeed, conf.Problem, maxIterations, conf.MaxSize);
+        }
+      }
+    }
+
+    #endregion
+
+    #region helpers
+    private IEnumerable<Configuration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
+      int nReps,
+      IEnumerable<int> maxSizes
+      ) {
+      var seedRand = new Random(randSeed);
+      // the problem seed is the same for all configuratons 
+      // this guarantees that we solve the _same_ problem each time
+      // with different solvers and multiple repetitions
+      var problemSeed = randSeed;
+      for (int i = 0; i < nReps; i++) {
+        // in each repetition use the same random seed for all solver configuratons
+        // do nReps with different seeds for each configuration
+        var solverSeed = seedRand.Next();
+        foreach (var maxSize in maxSizes) {
+          yield return new Configuration {
+            MaxSize = maxSize,
+            Problem = problemFactory(problemSeed),
+            RandSeed = solverSeed
+          };
+        }
+      }
+    }
+
+    private static void RunSeqSolvFuncApproxForProblem(
+      int randSeed,
+      IProblem problem,
+      int maxIters,
+      int maxSize
+      ) {
+      var solver = new SequentialSearch(problem, maxSize, new Random(randSeed), 0,
+        new GenericFunctionApproximationGrammarPolicy(problem, true));
+      var problemName = problem.GetType().Name;
+      var bestKnownQuality = problem.BestKnownQuality(maxSize);
+      RunSolver(solver, problemName, bestKnownQuality, maxIters, maxSize);
+    }
+
+    private static void RunSolver(ISolver solver, string problemName, double bestKnownQuality, int maxIters, int maxSize) {
+      int iterations = 0;
+      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
+      var gpName = solver.GetType().Name;
+      solver.SolutionEvaluated += (sentence, quality) => {
+        iterations++;
+        globalStatistics.AddSentence(sentence, quality);
+
+        if (iterations % 1000 == 0) {
+          Console.WriteLine("\"{0,25}\" {1} \"{2,25}\" {3}", gpName, maxSize, problemName, globalStatistics);
+        }
+      };
+
+      solver.Run(maxIters);
+    }
+    #endregion
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/Test.csproj
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/Test.csproj	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/Test.csproj	(revision 12290)
@@ -0,0 +1,134 @@
+﻿<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{8B4C199E-C08B-479E-9B82-6AE5B113459E}</ProjectGuid>
+    <OutputType>Library</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>HeuristicLab.Problems.GrammaticalOptimization.Test</RootNamespace>
+    <AssemblyName>Test</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <ProjectTypeGuids>{3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}</ProjectTypeGuids>
+    <VisualStudioVersion Condition="'$(VisualStudioVersion)' == ''">10.0</VisualStudioVersion>
+    <VSToolsPath Condition="'$(VSToolsPath)' == ''">$(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)</VSToolsPath>
+    <ReferencePath>$(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages</ReferencePath>
+    <IsCodedUITest>False</IsCodedUITest>
+    <TestProjectType>UnitTest</TestProjectType>
+    <TargetFrameworkProfile />
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+    <Prefer32Bit>false</Prefer32Bit>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.Core">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+  </ItemGroup>
+  <Choose>
+    <When Condition="('$(VisualStudioVersion)' == '10.0' or '$(VisualStudioVersion)' == '') and '$(TargetFrameworkVersion)' == 'v3.5'">
+      <ItemGroup>
+        <Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework, Version=10.1.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL" />
+      </ItemGroup>
+    </When>
+    <Otherwise>
+      <ItemGroup>
+        <Reference Include="Microsoft.VisualStudio.QualityTools.UnitTestFramework" />
+      </ItemGroup>
+    </Otherwise>
+  </Choose>
+  <ItemGroup>
+    <Compile Include="RunMctsExperiments.cs" />
+    <Compile Include="RunDemo.cs" />
+    <Compile Include="RunSequentialSolverFuncApproxExperiments.cs" />
+    <Compile Include="RunBaselineExperiments.cs" />
+    <Compile Include="RunGpExperiments.cs" />
+    <Compile Include="TestCanonicalExpressions.cs" />
+    <Compile Include="TestSymbRegInstances.cs" />
+    <Compile Include="TestSequence.cs" />
+    <Compile Include="TestBanditPolicies.cs" />
+    <Compile Include="TestInstances.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="TestSolvers.cs" />
+    <Compile Include="TestTunedSettings.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.Bandits\HeuristicLab.Algorithms.Bandits.csproj">
+      <Project>{24408f7d-ee0f-4886-a08b-ec324d662e47}</Project>
+      <Name>HeuristicLab.Algorithms.Bandits</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.GeneticProgramming\HeuristicLab.Algorithms.GeneticProgramming.csproj">
+      <Project>{14BEC23F-63FD-4954-B8AE-E2F4962E9B57}</Project>
+      <Name>HeuristicLab.Algorithms.GeneticProgramming</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Algorithms.GrammaticalOptimization\HeuristicLab.Algorithms.GrammaticalOptimization.csproj">
+      <Project>{eea07488-1a51-412a-a52c-53b754a628b3}</Project>
+      <Name>HeuristicLab.Algorithms.GrammaticalOptimization</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Common\HeuristicLab.Common.csproj">
+      <Project>{3A2FBBCB-F9DF-4970-87F3-F13337D941AD}</Project>
+      <Name>HeuristicLab.Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Distributions\HeuristicLab.Distributions.csproj">
+      <Project>{31171165-E16F-4A1A-A8AB-25C6AB3A71B9}</Project>
+      <Name>HeuristicLab.Distributions</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.Bandits\HeuristicLab.Problems.Bandits.csproj">
+      <Project>{E8138227-0C64-4E85-B676-21D3B97F254F}</Project>
+      <Name>HeuristicLab.Problems.Bandits</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization.SymbReg\HeuristicLab.Problems.GrammaticalOptimization.SymbReg.csproj">
+      <Project>{17a7a380-86ce-482d-8d22-cbd70cc97f0d}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization.SymbReg</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Problems.GrammaticalOptimization\HeuristicLab.Problems.GrammaticalOptimization.csproj">
+      <Project>{cb9dccf6-667e-4a13-b82d-dbd6b45a045e}</Project>
+      <Name>HeuristicLab.Problems.GrammaticalOptimization</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Choose>
+    <When Condition="'$(VisualStudioVersion)' == '10.0' And '$(IsCodedUITest)' == 'True'">
+      <ItemGroup>
+        <Reference Include="Microsoft.VisualStudio.QualityTools.CodedUITestFramework, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+          <Private>False</Private>
+        </Reference>
+        <Reference Include="Microsoft.VisualStudio.TestTools.UITest.Common, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+          <Private>False</Private>
+        </Reference>
+        <Reference Include="Microsoft.VisualStudio.TestTools.UITest.Extension, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+          <Private>False</Private>
+        </Reference>
+        <Reference Include="Microsoft.VisualStudio.TestTools.UITesting, Version=10.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL">
+          <Private>False</Private>
+        </Reference>
+      </ItemGroup>
+    </When>
+  </Choose>
+  <Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestBanditPolicies.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestBanditPolicies.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestBanditPolicies.cs	(revision 12290)
@@ -0,0 +1,256 @@
+﻿using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Globalization;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.Models;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  [TestClass]
+  public class TestBanditPolicies {
+    [TestMethod]
+    public void ComparePoliciesForGaussianUnknownVarianceBandit() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+      var randSeed = 31415;
+      var nArms = 20;
+
+      // ThresholdAscent only works for rewards in [0..1] so far
+
+      Console.WriteLine("Thompson (Gaussian est variance)"); TestPolicyGaussianUnknownVariance(randSeed, nArms, new GenericThompsonSamplingPolicy(new GaussianModel(0, 1, 1, 1)));
+      Console.WriteLine("Thompson (Gaussian fixed variance)"); TestPolicyGaussianUnknownVariance(randSeed, nArms, new GenericThompsonSamplingPolicy(new GaussianModel(0, 1, 0.1)));
+      Console.WriteLine("GaussianThompson (compat)"); TestPolicyGaussianUnknownVariance(randSeed, nArms, new GaussianThompsonSamplingPolicy(true));
+      Console.WriteLine("GaussianThompson"); TestPolicyGaussianUnknownVariance(randSeed, nArms, new GaussianThompsonSamplingPolicy());
+      Console.WriteLine("UCBNormal"); TestPolicyGaussianUnknownVariance(randSeed, nArms, new UCBNormalPolicy());
+      Console.WriteLine("Random"); TestPolicyGaussianUnknownVariance(randSeed, nArms, new RandomPolicy());
+
+    }
+
+
+    [TestMethod]
+    public void ComparePoliciesForBernoulliBandit() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+      var randSeed = 31415;
+      var nArms = 20;
+      //Console.WriteLine("Exp3 (gamma=0.01)");
+      //TestPolicyBernoulli(globalRand, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 1));
+      //Console.WriteLine("Exp3 (gamma=0.05)");
+      //estPolicyBernoulli(globalRand, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 1));
+      Console.WriteLine("Thompson (Bernoulli)"); TestPolicyBernoulli(randSeed, nArms, new BernoulliThompsonSamplingPolicy());
+      Console.WriteLine("Generic Thompson (Bernoulli)"); TestPolicyBernoulli(randSeed, nArms, new GenericThompsonSamplingPolicy(new BernoulliModel()));
+      Console.WriteLine("Random");
+      TestPolicyBernoulli(randSeed, nArms, new RandomPolicy());
+      Console.WriteLine("UCB1");
+      TestPolicyBernoulli(randSeed, nArms, new UCB1Policy());
+      Console.WriteLine("UCB1Tuned");
+      TestPolicyBernoulli(randSeed, nArms, new UCB1TunedPolicy());
+      Console.WriteLine("UCB1Normal");
+      TestPolicyBernoulli(randSeed, nArms, new UCBNormalPolicy());
+      Console.WriteLine("Eps(0.01)");
+      TestPolicyBernoulli(randSeed, nArms, new EpsGreedyPolicy(0.01));
+      Console.WriteLine("Eps(0.05)");
+      TestPolicyBernoulli(randSeed, nArms, new EpsGreedyPolicy(0.05));
+      //Console.WriteLine("Eps(0.1)");
+      //TestPolicyBernoulli(randSeed, nArms, new EpsGreedyPolicy(0.1));
+      //Console.WriteLine("Eps(0.2)");
+      //TestPolicyBernoulli(randSeed, nArms, new EpsGreedyPolicy(0.2));
+      //Console.WriteLine("Eps(0.5)");
+      //TestPolicyBernoulli(randSeed, nArms, new EpsGreedyPolicy(0.5));
+      Console.WriteLine("UCT(0.1)"); TestPolicyBernoulli(randSeed, nArms, new UCTPolicy(0.1));
+      Console.WriteLine("UCT(0.5)"); TestPolicyBernoulli(randSeed, nArms, new UCTPolicy(0.5));
+      Console.WriteLine("UCT(1)  "); TestPolicyBernoulli(randSeed, nArms, new UCTPolicy(1));
+      Console.WriteLine("UCT(2)  "); TestPolicyBernoulli(randSeed, nArms, new UCTPolicy(2));
+      Console.WriteLine("UCT(5)  "); TestPolicyBernoulli(randSeed, nArms, new UCTPolicy(5));
+      Console.WriteLine("BoltzmannExploration(0.1)"); TestPolicyBernoulli(randSeed, nArms, new BoltzmannExplorationPolicy(0.1));
+      Console.WriteLine("BoltzmannExploration(0.5)"); TestPolicyBernoulli(randSeed, nArms, new BoltzmannExplorationPolicy(0.5));
+      Console.WriteLine("BoltzmannExploration(1)  "); TestPolicyBernoulli(randSeed, nArms, new BoltzmannExplorationPolicy(1));
+      Console.WriteLine("BoltzmannExploration(10) "); TestPolicyBernoulli(randSeed, nArms, new BoltzmannExplorationPolicy(10));
+      Console.WriteLine("BoltzmannExploration(100)"); TestPolicyBernoulli(randSeed, nArms, new BoltzmannExplorationPolicy(100));
+      Console.WriteLine("ChernoffIntervalEstimationPolicy(0.01)"); TestPolicyBernoulli(randSeed, nArms, new ChernoffIntervalEstimationPolicy(0.01));
+      Console.WriteLine("ChernoffIntervalEstimationPolicy(0.05)"); TestPolicyBernoulli(randSeed, nArms, new ChernoffIntervalEstimationPolicy(0.05));
+      Console.WriteLine("ChernoffIntervalEstimationPolicy(0.1) "); TestPolicyBernoulli(randSeed, nArms, new ChernoffIntervalEstimationPolicy(0.1));
+
+      // not applicable to bernoulli rewards
+      //Console.WriteLine("ThresholdAscent(10, 0.01)  "); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 10, 0.01));
+      //Console.WriteLine("ThresholdAscent(10, 0.05)  "); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 10, 0.05));
+      //Console.WriteLine("ThresholdAscent(10, 0.1)   "); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 10, 0.1));
+      //Console.WriteLine("ThresholdAscent(100, 0.01) "); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 100, 0.01));
+      //Console.WriteLine("ThresholdAscent(100, 0.05) "); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 100, 0.05));
+      //Console.WriteLine("ThresholdAscent(100, 0.1)  "); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 100, 0.1));
+      //Console.WriteLine("ThresholdAscent(1000, 0.01)"); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 1000, 0.01));
+      //Console.WriteLine("ThresholdAscent(1000, 0.05)"); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 1000, 0.05));
+      //Console.WriteLine("ThresholdAscent(1000, 0.1) "); TestPolicyBernoulli(globalRand, nArms, new ThresholdAscentPolicy(nArms, 1000, 0.1));
+    }
+
+    [TestMethod]
+    public void ComparePoliciesForGaussianBandit() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var randSeed = 31415;
+      var nArms = 20;
+      Console.WriteLine("Threshold Ascent (20)"); TestPolicyGaussian(randSeed, nArms, new ThresholdAscentPolicy(20, 0.01));
+      Console.WriteLine("Threshold Ascent (100)"); TestPolicyGaussian(randSeed, nArms, new ThresholdAscentPolicy(100, 0.01));
+      Console.WriteLine("Threshold Ascent (500)"); TestPolicyGaussian(randSeed, nArms, new ThresholdAscentPolicy(500, 0.01));
+      Console.WriteLine("Threshold Ascent (1000)"); TestPolicyGaussian(randSeed, nArms, new ThresholdAscentPolicy(1000, 0.01));
+      Console.WriteLine("Generic Thompson (Gaussian fixed var)"); TestPolicyGaussian(randSeed, nArms, new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 1)));
+      Console.WriteLine("Generic Thompson (Gaussian unknown var)"); TestPolicyGaussian(randSeed, nArms, new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 1, 1, 1)));
+      Console.WriteLine("Thompson (Gaussian orig)"); TestPolicyGaussian(randSeed, nArms, new GaussianThompsonSamplingPolicy(true));
+      Console.WriteLine("Thompson (Gaussian new)"); TestPolicyGaussian(randSeed, nArms, new GaussianThompsonSamplingPolicy());
+
+      /*
+      Console.WriteLine("Random"); TestPolicyNormal(randSeed, nArms, new RandomPolicy(new Random(seedForPolicy), nArms));
+      Console.WriteLine("UCB1"); TestPolicyNormal(randSeed, nArms, new UCB1Policy(nArms));
+      Console.WriteLine("UCB1Tuned"); TestPolicyNormal(randSeed, nArms, new UCB1TunedPolicy(nArms));
+      Console.WriteLine("UCB1Normal"); TestPolicyNormal(randSeed, nArms, new UCBNormalPolicy(nArms));
+      //Console.WriteLine("Exp3 (gamma=0.01)");
+      //TestPolicyNormal(randSeed, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 0.01));
+      //Console.WriteLine("Exp3 (gamma=0.05)");
+      //TestPolicyNormal(randSeed, nArms, new Exp3Policy(new Random(seedForPolicy), nArms, 0.05));
+      Console.WriteLine("Eps(0.01)"); TestPolicyNormal(randSeed, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.01));
+      Console.WriteLine("Eps(0.05)"); TestPolicyNormal(randSeed, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.05));
+      //Console.WriteLine("Eps(0.1)");
+      //TestPolicyNormal(randSeed, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.1));
+      //Console.WriteLine("Eps(0.2)");
+      //TestPolicyNormal(randSeed, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.2));
+      //Console.WriteLine("Eps(0.5)");
+      //TestPolicyNormal(randSeed, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.5));
+      Console.WriteLine("UCT(0.1)"); TestPolicyNormal(randSeed, nArms, new UCTPolicy(nArms, 0.1));
+      Console.WriteLine("UCT(0.5)"); TestPolicyNormal(randSeed, nArms, new UCTPolicy(nArms, 0.5));
+      Console.WriteLine("UCT(1)  "); TestPolicyNormal(randSeed, nArms, new UCTPolicy(nArms, 1));
+      Console.WriteLine("UCT(2)  "); TestPolicyNormal(randSeed, nArms, new UCTPolicy(nArms, 2));
+      Console.WriteLine("UCT(5)  "); TestPolicyNormal(randSeed, nArms, new UCTPolicy(nArms, 5));
+      Console.WriteLine("BoltzmannExploration(0.1)"); TestPolicyNormal(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 0.1));
+      Console.WriteLine("BoltzmannExploration(0.5)"); TestPolicyNormal(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 0.5));
+      Console.WriteLine("BoltzmannExploration(1)  "); TestPolicyNormal(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 1));
+      Console.WriteLine("BoltzmannExploration(10) "); TestPolicyNormal(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 10));
+      Console.WriteLine("BoltzmannExploration(100)"); TestPolicyNormal(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 100));
+      Console.WriteLine("ChernoffIntervalEstimationPolicy(0.01)"); TestPolicyNormal(randSeed, nArms, new ChernoffIntervalEstimationPolicy(nArms, 0.01));
+      Console.WriteLine("ChernoffIntervalEstimationPolicy(0.05)"); TestPolicyNormal(randSeed, nArms, new ChernoffIntervalEstimationPolicy(nArms, 0.05));
+      Console.WriteLine("ChernoffIntervalEstimationPolicy(0.1) "); TestPolicyNormal(randSeed, nArms, new ChernoffIntervalEstimationPolicy(nArms, 0.1));      
+      Console.WriteLine("ThresholdAscent(10,0.01)  "); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 10, 0.01));
+      Console.WriteLine("ThresholdAscent(10,0.05)  "); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 10, 0.05));
+      Console.WriteLine("ThresholdAscent(10,0.1)   "); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 10, 0.1));
+      Console.WriteLine("ThresholdAscent(100,0.01) "); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 100, 0.01));
+      Console.WriteLine("ThresholdAscent(100,0.05) "); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 100, 0.05));
+      Console.WriteLine("ThresholdAscent(100,0.1)  "); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 100, 0.1));
+      Console.WriteLine("ThresholdAscent(1000,0.01)"); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 1000, 0.01));
+      Console.WriteLine("ThresholdAscent(1000,0.05)"); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 1000, 0.05));
+      Console.WriteLine("ThresholdAscent(1000,0.1) "); TestPolicyNormal(randSeed, nArms, new ThresholdAscentPolicy(nArms, 1000, 0.1));
+       */
+    }
+
+    [TestMethod]
+    public void ComparePoliciesForGaussianMixtureBandit() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+      var randSeed = 31415;
+      var nArms = 20;
+
+      Console.WriteLine("Generic Thompson (Gaussian Mixture)"); TestPolicyGaussianMixture(randSeed, nArms, new GenericThompsonSamplingPolicy(new GaussianMixtureModel()));
+      // Console.WriteLine("Threshold Ascent (20)"); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(20, 0.01));
+      // Console.WriteLine("Threshold Ascent (100)"); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(100, 0.01));
+      // Console.WriteLine("Threshold Ascent (500)"); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(500, 0.01));
+      // Console.WriteLine("Threshold Ascent (1000)"); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(1000, 0.01));
+      // Console.WriteLine("Thompson (Gaussian orig)"); TestPolicyGaussianMixture(randSeed, nArms, new GaussianThompsonSamplingPolicy(true));
+      // Console.WriteLine("Thompson (Gaussian new)"); TestPolicyGaussianMixture(randSeed, nArms, new GaussianThompsonSamplingPolicy());
+      // Console.WriteLine("Generic Thompson (Gaussian fixed variance)"); TestPolicyGaussianMixture(randSeed, nArms, new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 1, 0.1)));
+      // Console.WriteLine("Generic Thompson (Gaussian unknown variance)"); TestPolicyGaussianMixture(randSeed, nArms, new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 1, 1, 1)));
+
+      /*
+      Console.WriteLine("Random"); TestPolicyGaussianMixture(randSeed, nArms, new RandomPolicy(new Random(seedForPolicy), nArms));
+      Console.WriteLine("UCB1"); TestPolicyGaussianMixture(randSeed, nArms, new UCB1Policy(nArms));
+      Console.WriteLine("UCB1Tuned "); TestPolicyGaussianMixture(randSeed, nArms, new UCB1TunedPolicy(nArms));
+      Console.WriteLine("UCB1Normal"); TestPolicyGaussianMixture(randSeed, nArms, new UCBNormalPolicy(nArms));
+      Console.WriteLine("Eps(0.01) "); TestPolicyGaussianMixture(randSeed, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.01));
+      Console.WriteLine("Eps(0.05) "); TestPolicyGaussianMixture(randSeed, nArms, new EpsGreedyPolicy(new Random(seedForPolicy), nArms, 0.05));
+      Console.WriteLine("UCT(1)  "); TestPolicyGaussianMixture(randSeed, nArms, new UCTPolicy(nArms, 1));
+      Console.WriteLine("UCT(2)  "); TestPolicyGaussianMixture(randSeed, nArms, new UCTPolicy(nArms, 2));
+      Console.WriteLine("UCT(5)  "); TestPolicyGaussianMixture(randSeed, nArms, new UCTPolicy(nArms, 5));
+      Console.WriteLine("BoltzmannExploration(1)  "); TestPolicyGaussianMixture(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 1));
+      Console.WriteLine("BoltzmannExploration(10) "); TestPolicyGaussianMixture(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 10));
+      Console.WriteLine("BoltzmannExploration(100)"); TestPolicyGaussianMixture(randSeed, nArms, new BoltzmannExplorationPolicy(new Random(seedForPolicy), nArms, 100));
+
+      Console.WriteLine("ThresholdAscent(10,0.01)  "); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(nArms, 10, 0.01));
+      Console.WriteLine("ThresholdAscent(100,0.01) "); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(nArms, 100, 0.01));
+      Console.WriteLine("ThresholdAscent(1000,0.01)"); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(nArms, 1000, 0.01));
+      Console.WriteLine("ThresholdAscent(10000,0.01)"); TestPolicyGaussianMixture(randSeed, nArms, new ThresholdAscentPolicy(nArms, 10000, 0.01));
+       */
+    }
+
+
+    private void TestPolicyBernoulli(int randSeed, int nArms, IBanditPolicy policy) {
+      TestPolicy(randSeed, nArms, policy, (banditRandom, nActions) => new BernoulliBandit(banditRandom, nActions));
+    }
+    private void TestPolicyGaussian(int randSeed, int nArms, IBanditPolicy policy) {
+      TestPolicy(randSeed, nArms, policy, (banditRandom, nActions) => new TruncatedNormalBandit(banditRandom, nActions));
+    }
+    private void TestPolicyGaussianMixture(int randSeed, int nArms, IBanditPolicy policy) {
+      TestPolicy(randSeed, nArms, policy, (banditRandom, nActions) => new GaussianMixtureBandit(banditRandom, nActions));
+    }
+    private void TestPolicyGaussianUnknownVariance(int randSeed, int nArms, IBanditPolicy policy) {
+      TestPolicy(randSeed, nArms, policy, (banditRandom, nActions) => new GaussianBandit(banditRandom, nActions));
+    }
+
+
+    private void TestPolicy(int randSeed, int nArms, IBanditPolicy policy, Func<Random, int, IBandit> banditFactory) {
+      var maxIt = 1E5;
+      var reps = 10; // independent runs
+      var regretForIteration = new Dictionary<int, List<double>>();
+      var numberOfPullsOfSuboptimalArmsForExp = new Dictionary<int, double>();
+      var numberOfPullsOfSuboptimalArmsForMax = new Dictionary<int, double>();
+      var globalRandom = new Random(randSeed);
+      var banditRandom = new Random(globalRandom.Next()); // bandits must produce the same rewards for each test
+      var policyRandom = new Random(globalRandom.Next());
+
+      // calculate statistics
+      for (int r = 0; r < reps; r++) {
+        var nextLogStep = 1;
+        var b = banditFactory(banditRandom, nArms);
+        var totalRegret = 0.0;
+        var totalPullsOfSuboptimalArmsExp = 0.0;
+        var totalPullsOfSuboptimalArmsMax = 0.0;
+        var actionInfos = Enumerable.Range(0, nArms).Select(_ => policy.CreateActionInfo()).ToArray();
+        for (int i = 0; i <= maxIt; i++) {
+          var selectedAction = policy.SelectAction(policyRandom, actionInfos);
+          var reward = b.Pull(selectedAction);
+          actionInfos[selectedAction].UpdateReward(reward);
+
+          // collect stats
+          if (selectedAction != b.OptimalExpectedRewardArm) totalPullsOfSuboptimalArmsExp++;
+          if (selectedAction != b.OptimalMaximalRewardArm) totalPullsOfSuboptimalArmsMax++;
+          totalRegret += b.OptimalExpectedReward - reward;
+
+          if (i == nextLogStep) {
+            nextLogStep *= 2;
+            if (!regretForIteration.ContainsKey(i)) {
+              regretForIteration.Add(i, new List<double>());
+            }
+            regretForIteration[i].Add(totalRegret / i);
+
+            if (!numberOfPullsOfSuboptimalArmsForExp.ContainsKey(i)) {
+              numberOfPullsOfSuboptimalArmsForExp.Add(i, 0.0);
+            }
+            numberOfPullsOfSuboptimalArmsForExp[i] += totalPullsOfSuboptimalArmsExp;
+
+            if (!numberOfPullsOfSuboptimalArmsForMax.ContainsKey(i)) {
+              numberOfPullsOfSuboptimalArmsForMax.Add(i, 0.0);
+            }
+            numberOfPullsOfSuboptimalArmsForMax[i] += totalPullsOfSuboptimalArmsMax;
+          }
+        }
+      }
+      // print
+      foreach (var p in regretForIteration.Keys.OrderBy(k => k)) {
+        Console.WriteLine("iter {0,8} regret avg {1,7:F5} min {2,7:F5} max {3,7:F5} suboptimal pulls (exp) {4,7:F2} suboptimal pulls (max) {5,7:F2}",
+          p,
+          regretForIteration[p].Average(),
+          regretForIteration[p].Min(),
+          regretForIteration[p].Max(),
+          numberOfPullsOfSuboptimalArmsForExp[p] / (double)reps,
+          numberOfPullsOfSuboptimalArmsForMax[p] / (double)reps
+          );
+      }
+    }
+
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestCanonicalExpressions.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestCanonicalExpressions.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestCanonicalExpressions.cs	(revision 12290)
@@ -0,0 +1,119 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  [TestClass]
+  public class TestCanonicalExpressions {
+    [TestMethod]
+    public void TestFactorReordering() {
+      var extender = new ExpressionExtender();
+      Assert.AreEqual("a*b", extender.CanonicalRepresentation("b*a"));
+      Assert.AreEqual("a*b*c", extender.CanonicalRepresentation("b*a*c"));
+      Assert.AreEqual("a*b*E", extender.CanonicalRepresentation("b*a*E"));
+    }
+
+    [TestMethod]
+    public void TestDuplicateTerms() {
+      var extender = new ExpressionExtender();
+      Assert.AreEqual("a", extender.CanonicalRepresentation("a+a"));
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("b+a+b"));
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("b+a-b")); // even though b-b canceles should keep this as we might find a non-zero factor for b
+    }
+
+    [TestMethod]
+    public void TestSimpleTermReordering() {
+      var extender = new ExpressionExtender();
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("b+a"));
+      Assert.AreEqual("a+b+c", extender.CanonicalRepresentation("b+a+c"));
+      Assert.AreEqual("a+b+E", extender.CanonicalRepresentation("b+a+E"));
+    }
+
+    [TestMethod]
+    public void TestCombinedTermReordering() {
+      var extender = new ExpressionExtender();
+      Assert.AreEqual("a+b+a*b+E", extender.CanonicalRepresentation("b*a+b+a+E"));
+      Assert.AreEqual("a+b+b*c+E", extender.CanonicalRepresentation("c*b+b+a+E"));
+      Assert.AreEqual("a+b+b*c+a*b*c+E", extender.CanonicalRepresentation("c*b+b*a*c+b+a+E"));
+    }
+    [TestMethod]
+    public void TestNegativeAndReorderTerms() {
+      var extender = new ExpressionExtender();
+      // when using constant opt the negative sign is not necessary because a negative factor can be produced
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("b-a"));
+      Assert.AreEqual("a+b+c", extender.CanonicalRepresentation("b-a-c"));
+      Assert.AreEqual("a+b+E", extender.CanonicalRepresentation("b-a-E"));
+    }
+    [TestMethod]
+    public void TestFactorExpansion() {
+      var extender = new ExpressionExtender();
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("(b+a)"));
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("(b)+(a)"));
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("(b+(a))"));
+      Assert.AreEqual("a+b", extender.CanonicalRepresentation("((b)+a)"));
+
+      Assert.AreEqual("a+b+c", extender.CanonicalRepresentation("b-(a-c)"));
+      Assert.AreEqual("a+b+c", extender.CanonicalRepresentation("b-(a+c)"));
+      Assert.AreEqual("a+b+E", extender.CanonicalRepresentation("b-(a-E)"));
+
+      // when using constant opt the negative sign is not necessary because a negative factor can be produced
+      Assert.AreEqual("a*b+b*c", extender.CanonicalRepresentation("b*(c-a)"));
+      Assert.AreEqual("a*b+a*d+b*c+c*d", extender.CanonicalRepresentation("(b-d)*(c-a)"));
+      Assert.AreEqual("/(a+b)", extender.CanonicalRepresentation("c%c%(a+b)"));
+    }
+    [TestMethod]
+    public void TestDivisionExpansion() {
+      var extender = new ExpressionExtender();
+      Assert.AreEqual("a*/c+b*/c", extender.CanonicalRepresentation("(b+a)%c"));
+      Assert.AreEqual("a*/c*/d+b*/c*/d", extender.CanonicalRepresentation("(b+a)%(d*c)"));
+      Assert.AreEqual("a*/(c+d)+b*/(c+d)", extender.CanonicalRepresentation("(b-a)%(d-c)"));
+      Assert.AreEqual("a*b*/(c+d)", extender.CanonicalRepresentation("(b*a)%(d-c)"));
+
+      Assert.AreEqual("a*b*/(a+b)*/(c+d)", extender.CanonicalRepresentation("(b*a)%(d-c)%(a+b)"));
+      Assert.AreEqual("a*b*/(c+d)*/(a*/e+b*/e)", extender.CanonicalRepresentation("((b*a)%(d-c))%((a+b)%e)"));
+      // a*b*e%(c+d)%(a+b)
+    }
+    [TestMethod]
+    public void TestDivisionCancellation() {
+      var extender = new ExpressionExtender();
+      Assert.AreEqual("1", extender.CanonicalRepresentation("a%a"));
+      Assert.AreEqual("a", extender.CanonicalRepresentation("a*a%a"));
+      Assert.AreEqual("/a", extender.CanonicalRepresentation("(a%a)%a"));
+      Assert.AreEqual("/a", extender.CanonicalRepresentation("a%a%a"));
+      Assert.AreEqual("a", extender.CanonicalRepresentation("a%(a%a)"));
+      Assert.AreEqual("1", extender.CanonicalRepresentation("(a+b)%(b+a)"));
+      Assert.AreEqual("/a+/b", extender.CanonicalRepresentation("(a+b)%(a*b)"));
+      Assert.AreEqual("a*/(a*c*/b+e*/d*/f)+b*/(a*c*/b+e*/d*/f)", extender.CanonicalRepresentation("(a+b)%(a%b*c+e%f%d)"));
+      Assert.AreEqual("1", extender.CanonicalRepresentation("(a%a%a+b%b%b)%(a%a*a%a%a+b%b*b%b%b)"));
+      Assert.AreEqual("1", extender.CanonicalRepresentation("(a%(a%a)+b%(b%b))%(a+b)"));
+    }
+
+    [TestMethod]
+    public void TestRandomExpressions() {
+      // samples sentences for the Tower dataset with the random MCTS policy 
+      // and evaluates the original and the extended expression
+      // the results must be the same
+
+      var problem = new SymbolicRegressionProblem(new Random(), "Tower");
+      var random = new Random(31415);
+      var solver = new SequentialSearch(problem, 30, random, 0, new RandomPolicy(problem, true));
+
+      var extender = new ExpressionExtender();
+
+      solver.SolutionEvaluated += (sentence, quality) => {
+        var canonicalSentence = extender.CanonicalRepresentation(sentence);
+
+        Assert.AreEqual(problem.SimpleEvaluate(sentence), problem.SimpleEvaluate(canonicalSentence), 1E-4, string.Format("{0} <> {1}", sentence, canonicalSentence));
+      };
+
+      solver.Run(10000);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestInstances.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestInstances.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestInstances.cs	(revision 12290)
@@ -0,0 +1,272 @@
+﻿using System;
+using System.Diagnostics;
+using System.Linq;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  [TestClass]
+  public class TestInstances {
+    [TestMethod]
+    public void TestGrammarFromString() {
+      {
+        var g = Grammar.FromString(@"
+
+G(S):
+  S -> aA | bB
+  A -> a | aA
+  B -> Bb | b
+");
+        Assert.AreEqual('S', g.SentenceSymbol);
+        Assert.AreEqual(2, g.TerminalSymbols.Count());
+        Assert.AreEqual(3, g.NonTerminalSymbols.Count());
+        Assert.IsTrue(g.TerminalSymbols.All(s => new char[] { 'a', 'b' }.Contains(s)));
+        Assert.IsTrue(g.NonTerminalSymbols.All(s => new char[] { 'S', 'A', 'B' }.Contains(s)));
+
+        Assert.AreEqual(2, g.GetAlternatives('S').Count());
+        Assert.AreEqual(2, g.GetAlternatives('A').Count());
+        Assert.AreEqual(2, g.GetAlternatives('B').Count());
+
+        Assert.IsTrue(g.GetAlternatives('S').Any(s => s.ToString() == "aA"));
+        Assert.IsTrue(g.GetAlternatives('S').Any(s => s.ToString() == "bB"));
+        Assert.IsTrue(g.GetAlternatives('A').Any(s => s.ToString() == "aA"));
+        Assert.IsTrue(g.GetAlternatives('A').Any(s => s.ToString() == "a"));
+        Assert.IsTrue(g.GetAlternatives('B').Any(s => s.ToString() == "Bb"));
+        Assert.IsTrue(g.GetAlternatives('B').Any(s => s.ToString() == "b"));
+
+        Assert.AreEqual(2, g.MinPhraseLength(new Sequence("S")));
+        Assert.AreEqual(short.MaxValue, g.MaxPhraseLength(new Sequence("S")));
+        Assert.AreEqual(1, g.MinPhraseLength(new Sequence("A")));
+        Assert.AreEqual(short.MaxValue, g.MaxPhraseLength(new Sequence("A")));
+        Assert.AreEqual(1, g.MinPhraseLength(new Sequence("B")));
+        Assert.AreEqual(short.MaxValue, g.MaxPhraseLength(new Sequence("B")));
+      }
+
+      {
+        var g = Grammar.FromString(@"
+
+G(S):
+  S -> sS
+");
+        Assert.AreEqual('S', g.SentenceSymbol);
+        Assert.AreEqual(1, g.TerminalSymbols.Count());
+        Assert.AreEqual(1, g.NonTerminalSymbols.Count());
+        Assert.IsTrue(g.TerminalSymbols.All(s => new char[] { 's' }.Contains(s)));
+        Assert.IsTrue(g.NonTerminalSymbols.All(s => new char[] { 'S' }.Contains(s)));
+
+        Assert.AreEqual(1, g.GetAlternatives('S').Count());
+
+        Assert.IsTrue(g.GetAlternatives('S').Any(s => s.ToString() == "sS"));
+
+        Assert.AreEqual(short.MaxValue, g.MinPhraseLength(new Sequence("S")));
+        Assert.AreEqual(short.MaxValue, g.MaxPhraseLength(new Sequence("S")));
+      }
+
+      {
+        var g = Grammar.FromString(@"
+
+G(S):
+S->sss|sS");
+        Assert.AreEqual('S', g.SentenceSymbol);
+        Assert.AreEqual(1, g.TerminalSymbols.Count());
+        Assert.AreEqual(1, g.NonTerminalSymbols.Count());
+        Assert.IsTrue(g.TerminalSymbols.All(s => new char[] { 's' }.Contains(s)));
+        Assert.IsTrue(g.NonTerminalSymbols.All(s => new char[] { 'S' }.Contains(s)));
+
+        Assert.AreEqual(2, g.GetAlternatives('S').Count());
+
+        Assert.IsTrue(g.GetAlternatives('S').Any(s => s.ToString() == "sss"));
+        Assert.IsTrue(g.GetAlternatives('S').Any(s => s.ToString() == "sS"));
+
+        Assert.AreEqual(3, g.MinPhraseLength(new Sequence("S")));
+        Assert.AreEqual(short.MaxValue, g.MaxPhraseLength(new Sequence("S")));
+        Assert.AreEqual(4, g.MinPhraseLength(new Sequence("sS")));
+        Assert.AreEqual(7, g.MinPhraseLength(new Sequence("sSS")));
+        Assert.AreEqual(short.MaxValue, g.MaxPhraseLength(new Sequence("sSS")));
+        Assert.AreEqual(3, g.MaxPhraseLength(new Sequence("sss")));
+        Assert.AreEqual(3, g.MinPhraseLength(new Sequence("sss")));
+      }
+
+      {
+        var g = Grammar.FromString(@"
+
+G(S):
+S->T|TS
+T-> a..z
+");
+        Assert.AreEqual('S', g.SentenceSymbol);
+        Assert.AreEqual(26, g.TerminalSymbols.Count());
+        Assert.AreEqual(2, g.NonTerminalSymbols.Count());
+        Assert.IsTrue(g.NonTerminalSymbols.All(s => new char[] { 'S', 'T' }.Contains(s)));
+
+        Assert.AreEqual(2, g.GetAlternatives('S').Count());
+
+        Assert.IsTrue(g.GetAlternatives('S').Any(s => s.ToString() == "T"));
+        Assert.IsTrue(g.GetAlternatives('S').Any(s => s.ToString() == "TS"));
+
+        Assert.AreEqual(1, g.MinPhraseLength(new Sequence("S")));
+        Assert.AreEqual(short.MaxValue, g.MaxPhraseLength(new Sequence("S")));
+        Assert.AreEqual(1, g.MinPhraseLength(new Sequence("T")));
+        Assert.AreEqual(1, g.MaxPhraseLength(new Sequence("T")));
+      }
+
+    }
+
+    [TestMethod]
+    public void TestRoyalSymbolProblem() {
+      var p = new RoyalSymbolProblem();
+      Console.WriteLine(p.Grammar);
+
+      Assert.AreEqual(1, p.Evaluate("ab"));
+      Assert.AreEqual(0, p.Evaluate("bb"));
+      Assert.AreEqual(2, p.Evaluate("aa"));
+      Assert.AreEqual(3, p.Evaluate("aaa"));
+    }
+
+
+    [TestMethod]
+    public void TestRoyalPairProblem() {
+      var p = new RoyalPairProblem();
+      Console.WriteLine(p.Grammar);
+
+      Assert.AreEqual(1, p.Evaluate("ab"));
+      Assert.AreEqual(0, p.Evaluate("bb"));
+      Assert.AreEqual(0, p.Evaluate("aa"));
+      Assert.AreEqual(0, p.Evaluate("aaa"));
+      Assert.AreEqual(2, p.Evaluate("aba"));
+      Assert.AreEqual(2, p.Evaluate("abba"));
+      Assert.AreEqual(3, p.Evaluate("abab"));
+    }
+
+    [TestMethod]
+    public void TestSantaFeAntProblem() {
+      var p = new SantaFeAntProblem();
+      Console.WriteLine(p.Grammar);
+
+      // one possible optimal solution: if-food-ahead (move) ( left left if-food-ahead (move) (right)) move right
+      Assert.AreEqual(89, p.Evaluate("?(m)(ll?(m)(r))mr"));
+    }
+
+    [TestMethod]
+    public void TestRoyalRoadProblem() {
+      var p = new RoyalRoadProblem();
+      Console.WriteLine(p.Grammar);
+
+    }
+    [TestMethod]
+    public void TestPalindromeProblem() {
+      Assert.AreEqual("aba", PalindromeProblem.LongestPalindromicSubstring("aba"));
+      Assert.AreEqual("aba", PalindromeProblem.LongestPalindromicSubstring("aaba"));
+      Assert.AreEqual("aba", PalindromeProblem.LongestPalindromicSubstring("abaa"));
+      Assert.AreEqual("aabaa", PalindromeProblem.LongestPalindromicSubstring("aaaabaa"));
+      Assert.AreEqual("aabaa", PalindromeProblem.LongestPalindromicSubstring("aabaaaa"));
+      Assert.AreEqual("baaaab", PalindromeProblem.LongestPalindromicSubstring("aabaaaab"));
+
+
+      var p = new PalindromeProblem();
+      Console.WriteLine(p.Grammar);
+
+      Assert.AreEqual(1, p.Evaluate("a"));
+      Assert.AreEqual(2, p.Evaluate("aa"));
+      Assert.AreEqual(3, p.Evaluate("aaa"));
+      Assert.AreEqual(3, p.Evaluate("aba"));
+      Assert.AreEqual(3, p.Evaluate("baba"));
+      Assert.AreEqual(3, p.Evaluate("abab"));
+      Assert.AreEqual(5, p.Evaluate("babab"));
+    }
+    [TestMethod]
+    public void TestHardPalindromeProblem() {
+      var p = new HardPalindromeProblem();
+      Console.WriteLine(p.Grammar);
+
+      Assert.AreEqual(1 + 1, p.Evaluate("a"));
+      Assert.AreEqual(2 + 1, p.Evaluate("aa"));
+      Assert.AreEqual(3 + 1, p.Evaluate("aaa"));
+      Assert.AreEqual(3 + 2, p.Evaluate("aba"));
+      Assert.AreEqual(3 + 2, p.Evaluate("baba"));
+      Assert.AreEqual(3 + 2, p.Evaluate("abab"));
+      Assert.AreEqual(5 + 2, p.Evaluate("babab"));
+      Assert.AreEqual(5 + 3, p.Evaluate("bacab"));
+    }
+
+    [TestMethod]
+    public void TestEvenParityProblem() {
+      var p = new EvenParityProblem();
+      Console.WriteLine(p.Grammar);
+
+      // a | b | c | d | res
+      // 0 | 0 | 0 | 0 | 1
+      // 0 | 0 | 0 | 1 | 0
+      // 0 | 0 | 1 | 0 | 0
+      // 0 | 0 | 1 | 1 | 1
+      // 0 | 1 | 0 | 0 | 0
+      // 0 | 1 | 0 | 1 | 1
+      // 0 | 1 | 1 | 0 | 1
+      // 0 | 1 | 1 | 1 | 0
+      // 1 | 0 | 0 | 0 | 0
+      // 1 | 0 | 0 | 1 | 1
+      // 1 | 0 | 1 | 0 | 1
+      // 1 | 0 | 1 | 1 | 0
+      // 1 | 1 | 0 | 0 | 1
+      // 1 | 1 | 0 | 1 | 0
+      // 1 | 1 | 1 | 0 | 0
+      // 1 | 1 | 1 | 1 | 1
+
+
+      Assert.AreEqual(8, p.Evaluate("a"));
+      Assert.AreEqual(8, p.Evaluate("b"));
+      Assert.AreEqual(8, p.Evaluate("c"));
+      Assert.AreEqual(8, p.Evaluate("d"));
+
+      Assert.AreEqual(8, p.Evaluate("a*b")); // a and b
+      Assert.AreEqual(8, p.Evaluate("a+b")); // a or b
+      Assert.AreEqual(8, p.Evaluate("!a*b+a*!b")); // a xor b
+      Assert.AreEqual(8, p.Evaluate("!(a*b)+(a*!b)")); // a xor b
+      Assert.AreEqual(16, p.Evaluate("a^b^c^d")); // a xor b
+    }
+
+    [TestMethod]
+    public void TestSymbolicRegressionPoly10Problem() {
+      var p = new SymbolicRegressionPoly10Problem();
+      Console.WriteLine(p.Grammar);
+
+      Assert.AreEqual(0.00319925877644673, p.Evaluate("a"), 1.0E-7);
+      Assert.AreEqual(0.00632775454329686, p.Evaluate("b"), 1.0E-7);
+      Assert.AreEqual(0.00569030954291292, p.Evaluate("c"), 1.0E-7);
+      Assert.AreEqual(3.32317092313123E-05, p.Evaluate("d"), 1.0E-7);
+      Assert.AreEqual(0.000562538829174826, p.Evaluate("e"), 1.0E-7);
+      Assert.AreEqual(3.29458942533772E-06, p.Evaluate("f"), 1.0E-7);
+      Assert.AreEqual(0.000193866006166768, p.Evaluate("g"), 1.0E-7);
+      Assert.AreEqual(2.83339004096246E-05, p.Evaluate("h"), 1.0E-7);
+      Assert.AreEqual(0.00291584574914239, p.Evaluate("i"), 1.0E-7);
+      Assert.AreEqual(0.00504038616701691, p.Evaluate("j"), 1.0E-7);
+
+      Assert.AreEqual(0.252718466940018, p.Evaluate("a*b"), 1.0E-7);
+      Assert.AreEqual(0.290635611728845, p.Evaluate("c*d"), 1.0E-7);
+      Assert.AreEqual(0.25737325167716, p.Evaluate("e*f"), 1.0E-7);
+
+      Assert.AreEqual(0.00173739472363473, p.Evaluate("b*c"), 1.0E-7);
+      Assert.AreEqual(3.15450564064128E-05, p.Evaluate("d*e"), 1.0E-7);
+
+      Assert.AreEqual(0.0943358163760454, p.Evaluate("a*g*i"), 1.0E-7);
+      Assert.AreEqual(0.116199534934045, p.Evaluate("c*f*j"), 1.0E-7);
+
+      Assert.AreEqual(0.824522210419616, p.Evaluate("a*b+c*d+e*f"), 1E-7);
+
+
+      Assert.AreEqual(1.0, p.Evaluate("a*b+c*d+e*f+a*g*i+c*f*j"), 1.0E-7);
+    }
+
+    [TestMethod]
+    public void TestRoyalTreeProblem() {
+      var p = new RoyalTreeProblem(7);
+      Console.WriteLine(p.Grammar);
+
+      // examples from The Royal Tree Problem, a Benchmark for Single and Multi-population Genetic Programming (Punch, Zongker, Goodman)
+      Assert.AreEqual(4, p.Evaluate("ax"), 1.0E-7);
+      Assert.AreEqual(384.0, p.Evaluate("cbaxaxbaxaxbaxax"), 1.0E-7);
+      Assert.AreEqual(128.66, p.Evaluate("cbaxaxbaxaxbxx"), 1.0E-7);
+      Assert.AreEqual(64.66, p.Evaluate("cbaxaxxx"), 1.0E-7);
+
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSequence.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSequence.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSequence.cs	(revision 12290)
@@ -0,0 +1,325 @@
+﻿using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Globalization;
+using HeuristicLab.Algorithms.Bandits;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  [TestClass]
+  public class TestSequence {
+
+    [TestMethod]
+    public void TestCtorAndProperties() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+      {
+        var s = new Sequence("A");
+        Assert.AreEqual(1, s.Length);
+        Assert.AreEqual(false, s.IsTerminal);
+        Assert.AreEqual(0, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+      }
+      {
+        var s = new Sequence("AB");
+        Assert.AreEqual(2, s.Length);
+        Assert.AreEqual(false, s.IsTerminal);
+        Assert.AreEqual(0, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+      }
+      {
+        var s = new Sequence("aAB");
+        Assert.AreEqual(3, s.Length);
+        Assert.AreEqual(false, s.IsTerminal);
+        Assert.AreEqual(1, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+      }
+      {
+        var s = new Sequence("aaaaA");
+        Assert.AreEqual(5, s.Length);
+        Assert.AreEqual(false, s.IsTerminal);
+        Assert.AreEqual(4, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+      }
+      {
+        var s = new Sequence("aaaa");
+        Assert.AreEqual(4, s.Length);
+        Assert.AreEqual(true, s.IsTerminal);
+        Assert.AreEqual(-1, s.FirstNonTerminalIndex);
+        try {
+          var nt = s.FirstNonTerminal;
+          Assert.Fail();
+        } catch (IndexOutOfRangeException e) {
+        }
+      }
+      {
+        var s = new Sequence("aaaaA");
+        var t = new Sequence(s);
+        Assert.AreEqual(5, t.Length);
+        Assert.AreEqual(false, t.IsTerminal);
+        Assert.AreEqual(4, t.FirstNonTerminalIndex);
+        Assert.AreEqual('A', t.FirstNonTerminal);
+
+        t.ReplaceAt(0, 1, new Sequence("Z"));
+        Assert.AreEqual(5, t.Length);
+        Assert.AreEqual(false, t.IsTerminal);
+        Assert.AreEqual(0, t.FirstNonTerminalIndex);
+        Assert.AreEqual('Z', t.FirstNonTerminal);
+
+        // s must not be changed
+        Assert.AreEqual(5, s.Length);
+        Assert.AreEqual(false, s.IsTerminal);
+        Assert.AreEqual(4, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+      }
+    }
+
+    [TestMethod]
+    public void TestReplaceAt() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+      {
+        var s = new Sequence("A");
+        var t = new Sequence("B");
+
+        s.ReplaceAt(0, 1, t);
+        Assert.AreEqual('B', s.FirstNonTerminal);
+        Assert.AreEqual(1, s.Length);
+      }
+      {
+        var s = new Sequence("A");
+        var t = new Sequence("b");
+
+        s.ReplaceAt(0, 1, t);
+        Assert.AreEqual(-1, s.FirstNonTerminalIndex);
+        Assert.AreEqual(1, s.Length);
+      }
+      {
+        var s = new Sequence("a");
+        var t = new Sequence("B");
+
+        s.ReplaceAt(0, 1, t);
+        Assert.AreEqual('B', s.FirstNonTerminal);
+        Assert.AreEqual(1, s.Length);
+      }
+      {
+        var s = new Sequence("AB");
+        var t = new Sequence("a");
+
+        s.ReplaceAt(0, 1, t);
+        Assert.AreEqual(1, s.FirstNonTerminalIndex);
+        Assert.AreEqual('B', s.FirstNonTerminal);
+        Assert.AreEqual(2, s.Length);
+      }
+      {
+        var s = new Sequence("AB");
+        var t = new Sequence("aa");
+
+        s.ReplaceAt(0, 1, t);
+        Assert.AreEqual(2, s.FirstNonTerminalIndex);
+        Assert.AreEqual('B', s.FirstNonTerminal);
+        Assert.AreEqual(3, s.Length);
+        Assert.AreEqual("aaB", s.ToString());
+      }
+      {
+        var s = new Sequence("AB");
+        var t = new Sequence("bb");
+
+        s.ReplaceAt(1, 1, t);
+        Assert.AreEqual(0, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+        Assert.AreEqual(3, s.Length);
+        Assert.AreEqual("Abb", s.ToString());
+      }
+      {
+        var s = new Sequence("AB");
+        var t = new Sequence("aA");
+
+        s.ReplaceAt(0, 1, t);
+        Assert.AreEqual(1, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+        Assert.AreEqual(3, s.Length);
+        Assert.AreEqual("aAB", s.ToString());
+      }
+      {
+        var s = new Sequence("AaaB");
+        var t = new Sequence("a");
+
+        s.ReplaceAt(0, 3, t);
+        Assert.AreEqual(1, s.FirstNonTerminalIndex);
+        Assert.AreEqual('B', s.FirstNonTerminal);
+        Assert.AreEqual(2, s.Length);
+        Assert.AreEqual("aB", s.ToString());
+      }
+      {
+        var s = new Sequence("AaaB");
+        var t = new Sequence("aAa");
+
+        s.ReplaceAt(0, 3, t);
+        Assert.AreEqual(1, s.FirstNonTerminalIndex);
+        Assert.AreEqual('A', s.FirstNonTerminal);
+        Assert.AreEqual("aAaB", s.ToString());
+      }
+      {
+        var s = new Sequence("aaaA");
+        var t = new Sequence("aaa");
+
+        s.ReplaceAt(3, 1, t);
+        Assert.AreEqual(-1, s.FirstNonTerminalIndex);
+        Assert.AreEqual("aaaaaa", s.ToString());
+      }
+      {
+        var s = new Sequence("aaaA");
+        var t = new Sequence("aaa");
+
+        s.ReplaceAt(1, 3, t);
+        Assert.AreEqual(-1, s.FirstNonTerminalIndex);
+        Assert.AreEqual("aaaa", s.ToString());
+      }
+      {
+        var s = new Sequence("aaaA");
+        var t = new Sequence("aaa");
+
+        s.ReplaceAt(0, 4, t);
+        Assert.AreEqual(-1, s.FirstNonTerminalIndex);
+        Assert.AreEqual("aaa", s.ToString());
+      }
+      {
+        var s = new Sequence("aaaA");
+        var t = new Sequence("aAaAA");
+
+        s.ReplaceAt(0, 4, t);
+        Assert.AreEqual(1, s.FirstNonTerminalIndex);
+        Assert.AreEqual("aAaAA", s.ToString());
+      }
+
+      // testing length checks
+      {
+        var s = new Sequence("aA");
+        var t = new Sequence("aA");
+
+        try {
+          s.ReplaceAt(0, 4, t);
+          Assert.Fail();
+        } catch (ArgumentException) { }
+
+        s.ReplaceAt(0, 2, t); // should work
+
+        try {
+          s.ReplaceAt(0, 3, t);
+          Assert.Fail();
+        } catch (ArgumentException) { }
+
+        try {
+          s.ReplaceAt(1, 2, t);
+          Assert.Fail();
+        } catch (ArgumentException) { }
+
+        s.ReplaceAt(1, 1, new Sequence("A")); // should work
+
+        try {
+          s.ReplaceAt(-1, 2, t);
+          Assert.Fail();
+        } catch (ArgumentException) { }
+
+      }
+    }
+
+    [TestMethod]
+    public void TestSubsequence() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+      {
+        var s = new Sequence("AAaaaAA");
+        var sub = s.Subsequence(0, 7);
+
+        Assert.AreEqual(0, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("AAaaaAA", sub.ToString());
+      }
+      {
+        var s = new Sequence("AAaaaAA");
+        var sub = s.Subsequence(0, 3);
+
+        Assert.AreEqual(0, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("AAa", sub.ToString());
+      }
+      {
+        var s = new Sequence("AAaaaAA");
+        var sub = s.Subsequence(2, 3);
+
+        Assert.AreEqual(-1, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("aaa", sub.ToString());
+      }
+      {
+        var s = new Sequence("AAaaaAA");
+        var sub = s.Subsequence(2, 4);
+
+        Assert.AreEqual(3, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("aaaA", sub.ToString());
+      }
+      {
+        var s = new Sequence("AAaaaAA");
+        var sub = s.Subsequence(5, 2);
+
+        Assert.AreEqual(0, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("AA", sub.ToString());
+      }
+      {
+        var s = new Sequence("aaaAA");
+        var sub = s.Subsequence(2, 3);
+
+        Assert.AreEqual(1, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("aAA", sub.ToString());
+      }
+    }
+
+    [TestMethod]
+    public void TestReadonlySequence() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+      {
+        var s = new ReadonlySequence("AAaaaAA");
+        var sub = s.Subsequence(0, 7);
+
+        Assert.AreEqual(0, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("AAaaaAA", sub.ToString());
+      }
+      {
+        var s = new ReadonlySequence("AAaaaAA");
+        var sub = s.Subsequence(0, 3);
+
+        Assert.AreEqual(0, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("AAa", sub.ToString());
+
+        sub = sub.Subsequence(1, 2);
+        Assert.AreEqual(0, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("Aa", sub.ToString());
+      }
+      {
+        var s = new ReadonlySequence("AAaaaAA");
+        var sub = s.Subsequence(2, 3);
+
+        Assert.AreEqual(-1, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("aaa", sub.ToString());
+      }
+      {
+        var s = new ReadonlySequence("AAaaaAA");
+        var sub = s.Subsequence(2, 4);
+
+        Assert.AreEqual(3, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("aaaA", sub.ToString());
+      }
+      {
+        var s = new ReadonlySequence("AAaaaAA");
+        var sub = s.Subsequence(5, 2);
+
+        Assert.AreEqual(0, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("AA", sub.ToString());
+      }
+      {
+        var s = new ReadonlySequence("aaaAA");
+        var sub = s.Subsequence(2, 3);
+
+        Assert.AreEqual(1, sub.FirstNonTerminalIndex);
+        Assert.AreEqual("aAA", sub.ToString());
+      }
+
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSolvers.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSolvers.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSolvers.cs	(revision 12290)
@@ -0,0 +1,182 @@
+﻿using System;
+using HeuristicLab.Algorithms.GeneticProgramming;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  [TestClass]
+  public class TestSolvers {
+    [TestMethod]
+    public void TestExhaustiveBFS() {
+      {
+        // G(A): A -> l | r | m | ?(A)(A) | lA | rA | mA 
+        var prob = new SantaFeAntProblem();
+        var comb = 3;
+        TestBFS(prob, 1, comb);
+
+        comb = comb * 3 + 3;
+        TestBFS(prob, 2, comb);
+
+        comb = comb * 3 + 3;
+        TestBFS(prob, 3, comb);
+
+        comb = comb * 3 + 3;
+        TestBFS(prob, 4, comb);
+
+        comb = comb * 3 + 3;
+        TestBFS(prob, 5, comb);
+
+        comb = comb * 3 + 3;
+        TestBFS(prob, 6, comb);
+
+        comb = comb * 3 + 3 + (3 * 3); // additionally for ?(A)(A)
+        TestBFS(prob, 7, comb);
+      }
+      {
+        // G(E):
+        // E -> V | V+E | V-E | V*E | V/E | (E)
+        // V -> a .. j 
+        /* grammar has been change ... unit test not yet adapted 
+        var prob = new SymbolicRegressionPoly10Problem();
+        var comb = 10;
+        TestDFS(prob, 1, comb);
+        TestDFS(prob, 2, comb);
+        
+        comb = comb + 10 * 4 * comb + comb;
+        TestDFS(prob, 3, comb);
+        TestDFS(prob, 4, comb);
+        
+        comb = comb + 10 * 4 * comb + 10; // ((E)) 
+        TestDFS(prob, 5, comb);
+        TestDFS(prob, 6, comb);
+        
+        comb = comb + 10 * 4 * comb + 10; // (((E)))  */
+        // takes too long
+        //TestDFS(prob, 7, comb);
+        //TestDFS(prob, 8, comb);
+      }
+    }
+
+    private void TestBFS(IProblem prob, int len, int numExpectedSols) {
+      var solver = new ExhaustiveBreadthFirstSearch(prob, len);
+      int numSols = 0;
+
+      solver.SolutionEvaluated += (s, d) => { numSols++; };
+
+      solver.Run(int.MaxValue);
+      Assert.AreEqual(numExpectedSols, numSols);
+    }
+
+    [TestMethod]
+    public void TestExhaustiveDFS() {
+      {
+        // G(A): A -> l | r | m | ?(A)(A) | lA | rA | mA 
+        var prob = new SantaFeAntProblem();
+        var comb = 3;
+        TestDFS(prob, 1, comb);
+
+        comb = comb * 3 + 3;
+        TestDFS(prob, 2, comb);
+
+        comb = comb * 3 + 3;
+        TestDFS(prob, 3, comb);
+
+        comb = comb * 3 + 3;
+        TestDFS(prob, 4, comb);
+
+        comb = comb * 3 + 3;
+        TestDFS(prob, 5, comb);
+
+        comb = comb * 3 + 3;
+        TestDFS(prob, 6, comb);
+
+        comb = comb * 3 + 3 + (3 * 3); // additionally for ?(A)(A)
+        TestDFS(prob, 7, comb);
+      }
+      {
+        // G(E):
+        // E -> V | V+E | V-E | V*E | V/E | (E)
+        // V -> a .. j 
+        /* grammar has been change ... unit test not yet adapted 
+        var prob = new SymbolicRegressionPoly10Problem();
+        var comb = 10;
+        TestDFS(prob, 1, comb);
+        TestDFS(prob, 2, comb);
+
+        comb = comb + 10 * 4 * comb + comb;
+        TestDFS(prob, 3, comb);
+        TestDFS(prob, 4, comb);
+
+        comb = comb + 10 * 4 * comb + 10; // ((E)) 
+        TestDFS(prob, 5, comb);
+        TestDFS(prob, 6, comb);
+
+        comb = comb + 10 * 4 * comb + 10; // (((E))) */
+        // takes too long
+        //TestDFS(prob, 7, comb);
+        //TestDFS(prob, 8, comb);
+      }
+    }
+
+    private void TestDFS(IProblem prob, int len, int numExpectedSols) {
+      var solver = new ExhaustiveDepthFirstSearch(prob, len);
+      int numSols = 0;
+
+      solver.SolutionEvaluated += (s, d) => { numSols++; Console.WriteLine(s); };
+
+      solver.Run(int.MaxValue);
+      Assert.AreEqual(numExpectedSols, numSols);
+    }
+
+    [TestMethod]
+    public void TestStandardGP() {
+      var prob = new SymbolicRegressionPoly10Problem();
+      var rand = new Random(31415);
+      var sgp = new StandardGP(prob, rand);
+      sgp.PopulationSize = 10000;
+      sgp.MaxSolutionSize = 50;
+      sgp.MaxSolutionDepth = 20;
+
+      string bestSentence = string.Empty;
+      double bestQuality = double.NegativeInfinity;
+
+      sgp.FoundNewBestSolution += (sentence, quality) => {
+        Assert.Inconclusive(string.Format("{0:N3} {1}", quality, sentence));
+        bestSentence = sentence;
+        bestQuality = quality;
+      };
+
+      sgp.Run(100000);
+
+      Assert.AreEqual(1.0, bestQuality, 1E-6);
+      Assert.AreEqual("", bestSentence);
+    }
+
+    [TestMethod]
+    public void TestOSGP()
+    {
+      const int seed = 31415;
+      var prob = new SymbolicRegressionProblem(new Random(seed), "Tower");
+      var rand = new Random(31415);
+      var osgp = new OffspringSelectionGP(prob, rand);
+      osgp.PopulationSize = 500;
+      osgp.MaxSolutionSize = 100;
+      osgp.MaxSolutionDepth = 20;
+
+      string bestSentence = string.Empty;
+      double bestQuality = double.NegativeInfinity;
+
+      osgp.FoundNewBestSolution += (sentence, quality) => {
+        // Assert.Inconclusive(string.Format("{0:N3} {1}", quality, sentence));
+        bestSentence = sentence;
+        bestQuality = quality;
+      };
+
+      osgp.Run(100000);
+
+      Assert.AreEqual(1.0, bestQuality, 1E-6);
+      Assert.AreEqual("", bestSentence);
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSymbRegInstances.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSymbRegInstances.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestSymbRegInstances.cs	(revision 12290)
@@ -0,0 +1,46 @@
+﻿using System;
+using System.Linq;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.GeneticProgramming;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  // tests for the brige to the HL symb-reg instances
+  [TestClass]
+  public class TestSymbRegInstances {
+    [TestMethod]
+    public void TestGetDataDescriptors() {
+      var problem = new SymbolicRegressionProblem(new Random(), "Tower");
+      Assert.IsNotNull(problem);
+    }
+
+    [TestMethod]
+    public void TestConstantOptimization() {
+      double r2;
+      var problem = new SymbolicRegressionProblem(new Random(), "Tower");
+      Assert.AreEqual(problem.Evaluate("a*b"), problem.OptimizeConstantsAndEvaluate("a*b"));
+      Assert.AreEqual(problem.OptimizeConstantsAndEvaluate("a*b"), problem.Evaluate("a*b"));
+      Assert.AreEqual(problem.OptimizeConstantsAndEvaluate("0*a*b"), problem.Evaluate("a*b"), 1E-6);
+      Assert.AreEqual(problem.OptimizeConstantsAndEvaluate("0*a*b+1"), problem.Evaluate("a*b"), 1E-6);
+      Assert.IsTrue(problem.OptimizeConstantsAndEvaluate("0*a+b") >= problem.Evaluate("a+b"));
+      Assert.AreEqual(problem.OptimizeConstantsAndEvaluate("0*a+0*b"), problem.Evaluate("a+b"), 1E-6);
+      Assert.IsTrue(problem.OptimizeConstantsAndEvaluate("0*a+1*b") >= problem.Evaluate("a+b"));
+    }
+
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 8)] // 8 hours
+    public void TestSequentialSolverForTower() {
+      var problem = new SymbolicRegressionProblem(new Random(), "Tower");
+      var random = new Random(31415);
+      var solver = new SequentialSearch(problem, 50, random, 0, new GenericFunctionApproximationGrammarPolicy(problem, true));
+      solver.FoundNewBestSolution += (s, d) => {
+        Console.WriteLine("{0:F3} {1}", d, s);
+      };
+      solver.Run(100000);
+    }
+
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestTunedSettings.cs
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestTunedSettings.cs	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/Test/TestTunedSettings.cs	(revision 12290)
@@ -0,0 +1,433 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using HeuristicLab.Algorithms.Bandits;
+using HeuristicLab.Algorithms.Bandits.BanditPolicies;
+using HeuristicLab.Algorithms.Bandits.GrammarPolicies;
+using HeuristicLab.Algorithms.Bandits.Models;
+using HeuristicLab.Algorithms.GrammaticalOptimization;
+using HeuristicLab.Common;
+using HeuristicLab.Problems.GrammaticalOptimization.SymbReg;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using RandomPolicy = HeuristicLab.Algorithms.Bandits.BanditPolicies.RandomPolicy;
+
+namespace HeuristicLab.Problems.GrammaticalOptimization.Test {
+  [TestClass]
+  public class TestTunedSettings {
+    private const int randSeed = 31415;
+    internal class Configuration {
+      public IProblem Problem;
+      public int MaxSize;
+      public int RandSeed;
+
+      public override string ToString() {
+        return string.Format("{0} {1} {2}", RandSeed, Problem, MaxSize);
+      }
+    }
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 12)] // 12 hours
+    public void TestAllPoliciesArtificialAnt() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SantaFeAntProblem(), 
+      };
+
+      var policyFactories = new Func<IBanditPolicy>[]
+        {
+         () => new RandomPolicy(), 
+         () => new ActiveLearningPolicy(),  
+         // () => new EpsGreedyPolicy(0.01, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.05, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.1, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.2, (aInfo)=> aInfo.MaxReward, "max"), 
+         //() => new GaussianThompsonSamplingPolicy(), 
+         () => new GaussianThompsonSamplingPolicy(true), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1)), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1, 1)), 
+         //() => new BernoulliThompsonSamplingPolicy(),
+         () => new GenericThompsonSamplingPolicy(new BernoulliModel(1, 1)), 
+         () => new EpsGreedyPolicy(0.01), 
+         () => new EpsGreedyPolicy(0.05), 
+         () => new EpsGreedyPolicy(0.1), 
+         () => new EpsGreedyPolicy(0.2), 
+         () => new EpsGreedyPolicy(0.5), 
+         () => new UCTPolicy(0.01),
+         () => new UCTPolicy(0.05),
+         () => new UCTPolicy(0.1),
+         () => new UCTPolicy(0.5),
+         () => new UCTPolicy(1),
+         () => new UCTPolicy(2),
+         () => new UCTPolicy( 5),
+         () => new UCTPolicy( 10),
+         () => new ModifiedUCTPolicy(0.01),
+         () => new ModifiedUCTPolicy(0.05),
+         () => new ModifiedUCTPolicy(0.1),
+         () => new ModifiedUCTPolicy(0.5),
+         () => new ModifiedUCTPolicy(1),
+         () => new ModifiedUCTPolicy(2),
+         () => new ModifiedUCTPolicy( 5),
+         () => new ModifiedUCTPolicy( 10),
+         () => new UCB1Policy(), 
+         () => new UCB1TunedPolicy(), 
+         () => new UCBNormalPolicy(), 
+         () => new BoltzmannExplorationPolicy(1),
+         () => new BoltzmannExplorationPolicy(10),
+         () => new BoltzmannExplorationPolicy(20),
+         () => new BoltzmannExplorationPolicy(100),
+         () => new BoltzmannExplorationPolicy(200),
+         () => new BoltzmannExplorationPolicy(500),
+         () => new ChernoffIntervalEstimationPolicy( 0.01), 
+         () => new ChernoffIntervalEstimationPolicy( 0.05),
+         () => new ChernoffIntervalEstimationPolicy( 0.1),
+         () => new ChernoffIntervalEstimationPolicy( 0.2),
+         () => new ThresholdAscentPolicy(5, 0.01), 
+         () => new ThresholdAscentPolicy(5, 0.05),
+         () => new ThresholdAscentPolicy(5, 0.1),
+         () => new ThresholdAscentPolicy(5, 0.2),
+         () => new ThresholdAscentPolicy(10, 0.01), 
+         () => new ThresholdAscentPolicy(10, 0.05),
+         () => new ThresholdAscentPolicy(10, 0.1),
+         () => new ThresholdAscentPolicy(10, 0.2),
+         () => new ThresholdAscentPolicy(50, 0.01), 
+         () => new ThresholdAscentPolicy(50, 0.05),
+         () => new ThresholdAscentPolicy(50, 0.1),
+         () => new ThresholdAscentPolicy(50, 0.2),
+         () => new ThresholdAscentPolicy(100, 0.01), 
+         () => new ThresholdAscentPolicy(100, 0.05),
+         () => new ThresholdAscentPolicy(100, 0.1),
+         () => new ThresholdAscentPolicy(100, 0.2),
+         () => new ThresholdAscentPolicy(500, 0.01), 
+         () => new ThresholdAscentPolicy(500, 0.05),
+         () => new ThresholdAscentPolicy(500, 0.1),
+         () => new ThresholdAscentPolicy(500, 0.2),
+         () => new ThresholdAscentPolicy(5000, 0.01), 
+         () => new ThresholdAscentPolicy(10000, 0.01), 
+        };
+      var maxSizes = new int[] { 17 };  // necessary size for ant programm
+      int nReps = 20;
+      int maxIterations = 100000;
+      foreach (var instanceFactory in instanceFactories) {
+        var sumBestQ = 0.0;
+        var sumItersToBest = 0;
+        double fractionSolved = 0.0;
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          foreach (var policy in policyFactories) {
+            var prob = conf.Problem;
+            var maxLen = conf.MaxSize;
+            var rand = new Random(conf.RandSeed);
+
+            var solver = new SequentialSearch(prob, maxLen, rand, 0, new GenericGrammarPolicy(prob, policy(), true));
+            var problemName = prob.GetType().Name;
+            var policyName = policy().ToString();
+            double bestQ; int itersToBest;
+            RunSolver(solver, problemName, policyName, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
+            sumBestQ += bestQ;
+            sumItersToBest += itersToBest;
+            if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
+          }
+        }
+        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
+        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
+        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
+      }
+    }
+
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 30)] // 30 hours
+    public void TestAllPoliciesPoly10() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var policyFactories = new Func<IBanditPolicy>[]
+        {
+         () => new RandomPolicy(), 
+         () => new ActiveLearningPolicy(),  
+         // () => new EpsGreedyPolicy(0.01, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.05, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.1, (aInfo)=> aInfo.MaxReward, "max"), 
+         // () => new EpsGreedyPolicy(0.2, (aInfo)=> aInfo.MaxReward, "max"), 
+         //() => new GaussianThompsonSamplingPolicy(), 
+         () => new GaussianThompsonSamplingPolicy(true), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1)), 
+         () => new GenericThompsonSamplingPolicy(new GaussianModel(0.5, 10, 1, 1)), 
+         //() => new BernoulliThompsonSamplingPolicy(),
+         () => new GenericThompsonSamplingPolicy(new BernoulliModel(1, 1)), 
+         () => new EpsGreedyPolicy(0.01), 
+         () => new EpsGreedyPolicy(0.05), 
+         () => new EpsGreedyPolicy(0.1), 
+         () => new EpsGreedyPolicy(0.2), 
+         () => new EpsGreedyPolicy(0.5), 
+         () => new UCTPolicy(0.01),
+         () => new UCTPolicy(0.05),
+         () => new UCTPolicy(0.1),
+         () => new UCTPolicy(0.5),
+         () => new UCTPolicy(1),
+         () => new UCTPolicy(2),
+         () => new UCTPolicy( 5),
+         () => new UCTPolicy( 10),
+         () => new ModifiedUCTPolicy(0.01),
+         () => new ModifiedUCTPolicy(0.05),
+         () => new ModifiedUCTPolicy(0.1),
+         () => new ModifiedUCTPolicy(0.5),
+         () => new ModifiedUCTPolicy(1),
+         () => new ModifiedUCTPolicy(2),
+         () => new ModifiedUCTPolicy( 5),
+         () => new ModifiedUCTPolicy( 10),
+         () => new UCB1Policy(), 
+         () => new UCB1TunedPolicy(), 
+         () => new UCBNormalPolicy(), 
+         () => new BoltzmannExplorationPolicy(1),
+         () => new BoltzmannExplorationPolicy(10),
+         () => new BoltzmannExplorationPolicy(20),
+         () => new BoltzmannExplorationPolicy(100),
+         () => new BoltzmannExplorationPolicy(200),
+         () => new BoltzmannExplorationPolicy(500),
+         () => new ChernoffIntervalEstimationPolicy( 0.01), 
+         () => new ChernoffIntervalEstimationPolicy( 0.05),
+         () => new ChernoffIntervalEstimationPolicy( 0.1),
+         () => new ChernoffIntervalEstimationPolicy( 0.2),
+         () => new ThresholdAscentPolicy(5, 0.01), 
+         () => new ThresholdAscentPolicy(5, 0.05),
+         () => new ThresholdAscentPolicy(5, 0.1),
+         () => new ThresholdAscentPolicy(5, 0.2),
+         () => new ThresholdAscentPolicy(10, 0.01), 
+         () => new ThresholdAscentPolicy(10, 0.05),
+         () => new ThresholdAscentPolicy(10, 0.1),
+         () => new ThresholdAscentPolicy(10, 0.2),
+         () => new ThresholdAscentPolicy(50, 0.01), 
+         () => new ThresholdAscentPolicy(50, 0.05),
+         () => new ThresholdAscentPolicy(50, 0.1),
+         () => new ThresholdAscentPolicy(50, 0.2),
+         () => new ThresholdAscentPolicy(100, 0.01), 
+         () => new ThresholdAscentPolicy(100, 0.05),
+         () => new ThresholdAscentPolicy(100, 0.1),
+         () => new ThresholdAscentPolicy(100, 0.2),
+         () => new ThresholdAscentPolicy(500, 0.01), 
+         () => new ThresholdAscentPolicy(500, 0.05),
+         () => new ThresholdAscentPolicy(500, 0.1),
+         () => new ThresholdAscentPolicy(500, 0.2),
+         () => new ThresholdAscentPolicy(5000, 0.01), 
+         () => new ThresholdAscentPolicy(10000, 0.01), 
+        };
+      var maxSizes = new int[] { 23 };  // necessary size symb reg poly 10
+      int nReps = 20;
+      int maxIterations = 100000;
+      foreach (var instanceFactory in instanceFactories) {
+        var sumBestQ = 0.0;
+        var sumItersToBest = 0;
+        double fractionSolved = 0.0;
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          foreach (var policy in policyFactories) {
+            var prob = conf.Problem;
+            var maxLen = conf.MaxSize;
+            var rand = new Random(conf.RandSeed);
+
+            var solver = new SequentialSearch(prob, maxLen, rand, 0, new GenericGrammarPolicy(prob, policy(), true));
+            var problemName = prob.GetType().Name;
+            var policyName = policy().ToString();
+            double bestQ; int itersToBest;
+            RunSolver(solver, problemName, policyName, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
+            sumBestQ += bestQ;
+            sumItersToBest += itersToBest;
+            if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
+          }
+        }
+        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
+        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
+        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
+      }
+    }
+
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 30)] // 30 hours
+    public void TestAllSymbolicRegression() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        //(randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Nguyen F7", true),   very easy?!
+        //(randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Keijzer 6", true),  very easy?!
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Vladislavleva-4", 1.0, true),  // kommenda - const opt
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Spatial", 1.0, true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Friedman - II", 1.0, true), 
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Tower", 1.0, true), 
+      };
+
+      var policyFactories = new Func<IBanditPolicy>[]
+        {
+         () => new UCTPolicy(0.05),
+         () => new UCTPolicy(0.1),
+         () => new ModifiedUCTPolicy(0.01),
+         () => new ModifiedUCTPolicy(0.05),
+         () => new UCB1Policy(), 
+         () => new UCB1TunedPolicy(), 
+        };
+      var maxSizes = new int[] { 20 };  // default limit for all problems
+      int nReps = 20;
+      int maxIterations = 10000;
+      foreach (var instanceFactory in instanceFactories) {
+        var sumBestQ = 0.0;
+        var sumItersToBest = 0;
+        double fractionSolved = 0.0;
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          foreach (var policy in policyFactories) {
+            var prob = conf.Problem;
+            var maxLen = conf.MaxSize;
+            var rand = new Random(conf.RandSeed);
+
+            var solver = new SequentialSearch(prob, maxLen, rand, 0, new GenericGrammarPolicy(prob, policy(), true));
+            var problemName = prob.Name;
+            var policyName = policy().ToString();
+            double bestQ; int itersToBest;
+            RunSolver(solver, problemName, policyName, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
+            sumBestQ += bestQ;
+            sumItersToBest += itersToBest;
+            if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
+          }
+        }
+        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
+        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
+        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
+      }
+    }
+
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 12)] // 12 hours
+    // this configuration worked especially well in the experiments
+    public void TestPoly10WithOutConstantOpt() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionPoly10Problem(), 
+      };
+
+      var maxSizes = new int[] { 23 };
+      int nReps = 20;
+      int maxIterations = 100000;
+      foreach (var instanceFactory in instanceFactories) {
+        var sumBestQ = 0.0;
+        var sumItersToBest = 0;
+        double fractionSolved = 0.0;
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          var prob = conf.Problem;
+          var maxLen = conf.MaxSize;
+          var rand = new Random(conf.RandSeed);
+
+          var solver = new SequentialSearch(prob, maxLen, rand, 0,
+            new GenericFunctionApproximationGrammarPolicy(prob, true));
+
+          var problemName = prob.GetType().Name;
+          double bestQ; int itersToBest;
+          RunSolver(solver, problemName, string.Empty, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
+          sumBestQ += bestQ;
+          sumItersToBest += itersToBest;
+          if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
+        }
+        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
+        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
+        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
+      }
+    }
+
+    [TestMethod]
+    [Timeout(1000 * 60 * 60 * 12)] // 12 hours
+    // this configuration worked especially well in the experiments
+    public void TestPoly10WithConstantOpt() {
+      CultureInfo.DefaultThreadCurrentCulture = CultureInfo.InvariantCulture;
+
+      var instanceFactories = new Func<int, ISymbolicExpressionTreeProblem>[]
+      {
+        (randSeed) => (ISymbolicExpressionTreeProblem)new SymbolicRegressionProblem(new Random(randSeed), "Poly-10", 1.0, true ), 
+      };
+
+      var maxSizes = new int[] { 23 };
+      int nReps = 20;
+      int maxIterations = 100000;
+      foreach (var instanceFactory in instanceFactories) {
+        var sumBestQ = 0.0;
+        var sumItersToBest = 0;
+        double fractionSolved = 0.0;
+        foreach (var conf in GenerateConfigurations(instanceFactory, nReps, maxSizes)) {
+          var prob = conf.Problem;
+          var maxLen = conf.MaxSize;
+          var rand = new Random(conf.RandSeed);
+
+          var solver = new SequentialSearch(prob, maxLen, rand, 0,
+            new GenericFunctionApproximationGrammarPolicy(prob, true));
+
+          var problemName = prob.GetType().Name;
+          double bestQ; int itersToBest;
+          RunSolver(solver, problemName, string.Empty, 1.0, maxIterations, maxLen, out bestQ, out itersToBest);
+          sumBestQ += bestQ;
+          sumItersToBest += itersToBest;
+          if (bestQ.IsAlmost(1.0)) fractionSolved += 1.0 / nReps;
+        }
+        // Assert.AreEqual(0.85, fractionSolved, 1E-6);
+        // Assert.AreEqual(0.99438202247191, sumBestQ / nReps, 1E-6);
+        // Assert.AreEqual(5461.7, sumItersToBest / (double)nReps, 1E-6);
+      }
+    }
+
+    private IEnumerable<Configuration> GenerateConfigurations(Func<int, ISymbolicExpressionTreeProblem> problemFactory,
+     int nReps,
+     IEnumerable<int> maxSizes
+     ) {
+      var seedRand = new Random(randSeed);
+      // the problem seed is the same for all configuratons 
+      // this guarantees that we solve the _same_ problem each time
+      // with different solvers and multiple repetitions
+      var problemSeed = randSeed;
+      for (int i = 0; i < nReps; i++) {
+        // in each repetition use the same random seed for all solver configuratons
+        // do nReps with different seeds for each configuration
+        var solverSeed = seedRand.Next();
+        foreach (var maxSize in maxSizes) {
+          yield return new Configuration {
+            MaxSize = maxSize,
+            Problem = problemFactory(problemSeed),
+            RandSeed = solverSeed
+          };
+        }
+      }
+    }
+
+
+
+
+    private static void RunSolver(ISolver solver, string problemName, string policyName, double bestKnownQuality, int maxIters, int maxSize,
+      out double bestQ, out int itersToBest) {
+      int iterations = 0;
+      var globalStatistics = new SentenceSetStatistics(bestKnownQuality);
+      var solverName = solver.GetType().Name;
+      double bestQuality = double.NegativeInfinity;
+      int iterationsToBest = -1;
+      solver.SolutionEvaluated += (sentence, quality) => {
+        iterations++;
+        globalStatistics.AddSentence(sentence, quality);
+        if (quality > bestQuality) {
+          bestQuality = quality;
+          iterationsToBest = iterations;
+        }
+        if (iterations % 1000 == 0) {
+          Console.WriteLine("\"{0,25}\" \"{1,25}\" {2} \"{3,25}\" {4}", solverName, policyName, maxSize, problemName, globalStatistics);
+        }
+      };
+
+
+      solver.Run(maxIters);
+
+      bestQ = bestQuality;
+      itersToBest = iterationsToBest;
+    }
+  }
+}
Index: /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/local.testsettings
===================================================================
--- /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/local.testsettings	(revision 12290)
+++ /branches/HeuristicLab.Problems.GrammaticalOptimization-gkr/local.testsettings	(revision 12290)
@@ -0,0 +1,26 @@
+﻿<?xml version="1.0" encoding="UTF-8"?>
+<TestSettings name="local" id="72f27ca1-7110-4de3-8343-8801c6a3f97a" xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010">
+  <Description>These are default test settings for a local test run.</Description>
+  <Deployment enabled="false" />
+  <Execution hostProcessPlatform="MSIL">
+    <TestTypeSpecific>
+      <UnitTestRunConfig testTypeId="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b">
+        <AssemblyResolution>
+          <TestDirectory useLoadContext="true" />
+        </AssemblyResolution>
+      </UnitTestRunConfig>
+      <WebTestRunConfiguration testTypeId="4e7599fa-5ecb-43e9-a887-cd63cf72d207">
+        <Browser name="Internet Explorer 9.0" MaxConnections="6">
+          <Headers>
+            <Header name="User-Agent" value="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)" />
+            <Header name="Accept" value="*/*" />
+            <Header name="Accept-Language" value="{{$IEAcceptLanguage}}" />
+            <Header name="Accept-Encoding" value="GZIP" />
+          </Headers>
+        </Browser>
+      </WebTestRunConfiguration>
+    </TestTypeSpecific>
+    <AgentRule name="LocalMachineDefaultRole">
+    </AgentRule>
+  </Execution>
+</TestSettings>
