Index: /branches/2886_SymRegGrammarEnumeration/ExpressionClustering/ExpressionClustering.csproj
===================================================================
--- /branches/2886_SymRegGrammarEnumeration/ExpressionClustering/ExpressionClustering.csproj	(revision 15902)
+++ /branches/2886_SymRegGrammarEnumeration/ExpressionClustering/ExpressionClustering.csproj	(revision 15903)
@@ -56,4 +56,8 @@
       <HintPath>..\..\..\trunk\bin\HeuristicLab.Core.Views-3.3.dll</HintPath>
     </Reference>
+    <Reference Include="HeuristicLab.Data-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec" />
+    <Reference Include="HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4">
+      <HintPath>..\..\..\trunk\bin\HeuristicLab.Encodings.SymbolicExpressionTreeEncoding-3.4.dll</HintPath>
+    </Reference>
     <Reference Include="HeuristicLab.MainForm-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
@@ -66,4 +70,7 @@
     <Reference Include="HeuristicLab.Problems.DataAnalysis-3.4">
       <HintPath>..\..\..\trunk\bin\HeuristicLab.Problems.DataAnalysis-3.4.dll</HintPath>
+    </Reference>
+    <Reference Include="HeuristicLab.Problems.DataAnalysis.Symbolic-3.4">
+      <HintPath>..\..\..\trunk\bin\HeuristicLab.Problems.DataAnalysis.Symbolic-3.4.dll</HintPath>
     </Reference>
     <Reference Include="HeuristicLab.Visualization.ChartControlsExtensions-3.3, Version=3.3.0.0, Culture=neutral, PublicKeyToken=ba48961d6f65dcec, processorArchitecture=MSIL">
Index: /branches/2886_SymRegGrammarEnumeration/ExpressionClustering/Program.cs
===================================================================
--- /branches/2886_SymRegGrammarEnumeration/ExpressionClustering/Program.cs	(revision 15902)
+++ /branches/2886_SymRegGrammarEnumeration/ExpressionClustering/Program.cs	(revision 15903)
@@ -1,3 +1,4 @@
 ﻿using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Drawing;
@@ -6,13 +7,16 @@
 using HeuristicLab.Analysis;
 using HeuristicLab.Analysis.Views;
-using System.Windows.Forms;
-
+using HeuristicLab.Encodings.SymbolicExpressionTreeEncoding;
+using HeuristicLab.Problems.DataAnalysis;
+using HeuristicLab.Problems.DataAnalysis.Symbolic;
+
+// Evaluates sentences on randomly generated data
 namespace ExpressionClustering {
   class Program {
     private static readonly string folder = Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory);
     private static readonly string clusterFolder = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory), "clusters");
-    private static readonly string distinctSentencesFileName = Path.Combine(folder, @"distinctSentences.csv");
-    private static readonly string allSentencesFileName = Path.Combine(folder, "allSentences.csv");
-    private static readonly string outputFileName = Path.Combine(folder, "evaluations.csv.gz");
+    private static readonly string distinctSentencesFileName = Path.Combine(folder, @"distinctSentences_2018-04-13_09-52_TreeSize-10.csv");
+    private static readonly string allSentencesFileName = Path.Combine(folder, "allSentences_2018-04-13_09-52_TreeSize-10.csv");
+    private static readonly string outputFileName = Path.Combine(folder, "evaluations_2018-04-13_09-52_TreeSize-10.csv.gz");
     private static int N = 100;
     private static double[] evalBuf = new double[N];
@@ -22,5 +26,13 @@
     private static double max = +5.0;
     private static double[] xs = Enumerable.Range(1, N).Select(xi => ((double)xi / N) * (max - min) + min).ToArray(); // input
-    private static double[] ys = xs.Select(xi => 1.0 / (1 + Math.Pow(xi, -4))).ToArray(); // target (necessary for scaling and clustering 
+
+    private static double[] ys_pagie = xs.Select(xi => 1.0 / (1 + Math.Pow(xi, -4))).ToArray(); // a potential target (not used for search) 
+
+    // x³  * exp(-x) * cos(x) * sin(x) * (sin(x)² * cos(x) - 1)
+    // for keijzer x should be in scale 0 - 10 inclusive
+    private static double[] ys_keijzer4 = xs
+      .Select(xi => xi + 10.0) // scale 
+      .Select(xi => xi * xi * xi + Math.Exp(-xi) * Math.Cos(xi) * Math.Sin(xi) * (Math.Sin(xi) * Math.Sin(xi) * Math.Cos(xi) - 1))
+      .ToArray();
 
 
@@ -35,9 +47,6 @@
     static void Main(string[] args) {
 
-      var hash2Sentences = new Dictionary<string, List<string>>();
-      // for debugging only
+      var hash2Postfix = new Dictionary<string, List<string>>();
       var postfix2infix = new Dictionary<string, string>();
-
-
 
       // read all sentences and determine shortest sentences
@@ -49,16 +58,19 @@
           var line = reader.ReadLine();
           var toks = line.Split(';');
-          var hash = toks[2];
-          List<string> ls;
-          if (!hash2Sentences.TryGetValue(hash, out ls)) {
-            ls = new List<string>(1);
-            hash2Sentences.Add(hash, ls);
+          var hash = toks[0];
+          var length = toks[1];
+          var postfix = toks[2];
+          var infix = toks[3];
+          List<string> alternativesList;
+          if (!hash2Postfix.TryGetValue(hash, out alternativesList)) {
+            alternativesList = new List<string>(1);
+            hash2Postfix.Add(hash, alternativesList);
           }
-          ls.Add(toks[1]);
-          postfix2infix.Add(toks[1], toks[0]);
+          alternativesList.Add(postfix);
+          postfix2infix.Add(postfix, infix);
           nSentences++;
         }
 
-        Console.WriteLine("{0} {1}", nSentences, hash2Sentences.Count);
+        Console.WriteLine("{0} {1}", nSentences, hash2Postfix.Count);
         //Evaluate(toks[1], xs, evalBuf);
       }
@@ -66,19 +78,25 @@
       List<double[]> functions = new List<double[]>();
       List<string> sentences = new List<string>();
-      List<double> qualities = new List<double>();
-
-      foreach (var kvp in hash2Sentences) {
+      List<double[]> qualities = new List<double[]>(); // we might have multiple target functions to which we might compare
+
+      var ds = new Dataset(new string[] { "X" }, new IList[] { xs });
+      foreach (var kvp in hash2Postfix) {
         var ls = kvp.Value;
         var sentence = FindShortest(ls);
-        Evaluate(sentence, xs, evalBuf);
-        if (evalBuf.Any(ei => float.IsInfinity((float)ei) || float.IsNaN((float)ei))) {
+        //EvaluatePostfix(sentence, xs, evalBuf);
+        evalBuf = EvaluateInfix(postfix2infix[sentence], ds).ToArray();
+        if (evalBuf.Any(ei => double.IsInfinity(ei) || double.IsNaN(ei))) {
           Console.WriteLine("skipping {0} {1}", evalBuf.Average(), sentence);
         } else {
           try {
-            Scale(evalBuf, ys);
+            Scale(evalBuf);
             functions.Add((double[])evalBuf.Clone());
             sentences.Add(sentence);
-            HeuristicLab.Problems.DataAnalysis.OnlineCalculatorError error;
-            qualities.Add(HeuristicLab.Problems.DataAnalysis.OnlinePearsonsRSquaredCalculator.Calculate(evalBuf, ys, out error));
+            OnlineCalculatorError error;
+            var r2_pagie = OnlinePearsonsRSquaredCalculator.Calculate(evalBuf, ys_pagie, out error);
+            if (error != OnlineCalculatorError.None) r2_pagie = 0.0;
+            var r2_keijzer4 = OnlinePearsonsRSquaredCalculator.Calculate(evalBuf, ys_keijzer4, out error);
+            if (error != OnlineCalculatorError.None) r2_keijzer4 = 0.0;
+            qualities.Add(new double[] { r2_pagie, r2_keijzer4});
           } catch (ArgumentException e) {
             // scaling failed
@@ -87,38 +105,43 @@
       }
 
+
       List<int> clusters;
       List<double> distances;
-      Flann.FindClusters(functions, out clusters, out distances, 100);
-      
+      // DEACTIVATED FOR NOW -> USE LARGEVIS in R instead
+      // Flann.FindClusters(functions, out clusters, out distances, 100);
+      clusters = functions.Select(_ => 0).ToList();
+      distances = functions.Select(_ => 0.0).ToList();
+      // 
       // output all clusters and functions
       using (var writer = new StreamWriter(new System.IO.Compression.GZipStream(new FileStream(outputFileName, FileMode.OpenOrCreate), System.IO.Compression.CompressionMode.Compress))) {
         for (int i = 0; i < functions.Count; i++) {
-          writer.WriteLine("{0};{1};{2};{3};{4};{5}", clusters[i], distances[i], qualities[i], sentences[i], postfix2infix[sentences[i]], string.Join(";", functions[i].Select(fi => fi.ToString())));
+          writer.WriteLine("{0};{1};{2};{3};{4};{5}", clusters[i], distances[i], string.Join(";", qualities[i]), sentences[i], postfix2infix[sentences[i]], string.Join(";", functions[i].Select(fi => fi.ToString())));
         }
       }
-
-      var funClusters = functions.Zip(clusters, (f, c) => Tuple.Create(f, c)).GroupBy(t => t.Item2);
-      var dtView = new DataTableView();
-      dtView.Size = new Size(800, 600);
-
-      foreach (var funCluster in funClusters) {
-        // draw the functions for each cluster into a separate png
-        var dtName = string.Format("R² {0}", Enumerable.Range(0, qualities.Count).Where(idx => clusters[idx] == funCluster.Key).Select(idx => qualities[idx]).Average());
-        var dt = new DataTable(dtName, dtName);
-        var rows = new List<DataRow>();
-        int i = 0;
-        foreach (var fun in funCluster.Select(t => t.Item1)) {
-          var name = i.ToString();
-          var dr = new DataRow(name, name, fun);
-          rows.Add(dr);
-          i++;
-        }
-        dt.Rows.AddRange(rows);
-        dtView.Content = dt;
-        using (var bm = new Bitmap(800, 600)) {
-          dtView.DrawToBitmap(bm, new Rectangle(0, 0, 800, 600));
-          bm.Save(Path.Combine(clusterFolder, string.Format("cluster_{0,3}.png", funCluster.Key)));
-        }
-      }
+      // 
+      // var funClusters = functions.Zip(clusters, (f, c) => Tuple.Create(f, c)).GroupBy(t => t.Item2);
+      // var dtView = new DataTableView();
+      // dtView.Size = new Size(800, 600);
+      // 
+      // foreach (var funCluster in funClusters) {
+      //   // draw the functions for each cluster into a separate png
+      //   // var dtName = string.Format("R² {0}", Enumerable.Range(0, qualities.Count).Where(idx => clusters[idx] == funCluster.Key).Select(idx => qualities[idx]).Average());
+      //   var dtName = "Cluster";
+      //   var dt = new DataTable(dtName, dtName);
+      //   var rows = new List<DataRow>();
+      //   int i = 0;
+      //   foreach (var fun in funCluster.Select(t => t.Item1)) {
+      //     var name = i.ToString();
+      //     var dr = new DataRow(name, name, fun);
+      //     rows.Add(dr);
+      //     i++;
+      //   }
+      //   dt.Rows.AddRange(rows);
+      //   dtView.Content = dt;
+      //   using (var bm = new Bitmap(800, 600)) {
+      //     dtView.DrawToBitmap(bm, new Rectangle(0, 0, 800, 600));
+      //     bm.Save(Path.Combine(clusterFolder, string.Format("cluster_{0,3}.png", funCluster.Key)));
+      //   }
+      // }
     }
 
@@ -136,5 +159,26 @@
 
     #region evaluation
-    // linear scaling
+
+    // scaling to zero-mean unit variance
+    private static void Scale(double[] evalBuf) {
+      double mean;
+      double variance;
+      var max = evalBuf.Max();
+      for (int i = 0; i < evalBuf.Length; i++) {
+        evalBuf[i] /= max;
+      }
+
+      OnlineCalculatorError error, varError;
+      OnlineMeanAndVarianceCalculator.Calculate(evalBuf, out mean, out variance, out error, out varError);
+      if(error!=OnlineCalculatorError.None || varError != OnlineCalculatorError.None) {
+        throw new ArgumentException("Cannot scale vector");
+      }
+
+      for (int i = 0; i < evalBuf.Length; i++) {
+        evalBuf[i] = 1.0 / variance * evalBuf[i] + mean;
+      }
+    }
+
+    // linear scaling to match target
     private static void Scale(double[] evalBuf, double[] ys) {
       double alpha;
@@ -152,6 +196,15 @@
     }
 
+    // evaluates infix expressions (using the infix parser)
+    private static IEnumerable<double> EvaluateInfix(string infixExpr, Dataset ds) {
+      var parser = new HeuristicLab.Problems.DataAnalysis.Symbolic.InfixExpressionParser();
+      var tree = parser.Parse(infixExpr);
+      var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
+      return interpreter.GetSymbolicExpressionTreeValues(tree, ds, Enumerable.Range(0, ds.Rows));
+    }                                                 
+
+    /*
     // evaluates postfix expressions (only for a very specific format)
-    private static void Evaluate(string postfixExpr, double[] xs, double[] evalBuf) {
+    private static void EvaluatePostfix(string postfixExpr, double[] xs, double[] evalBuf) {
       int topOfStack = -1;
       Evaluate(postfixExpr, 0, xs, ref topOfStack);
@@ -159,4 +212,5 @@
     }
 
+    
     private static void Evaluate(string postfixExpr, int exprPos, double[] xs, ref int topOfStack) {
       while (exprPos < postfixExpr.Length) {
@@ -189,11 +243,20 @@
             }
           case 'c': {
-              // cos
-              exprPos += 4;
-              var a = stack[topOfStack];
-              for (int i = 0; i < N; i++) {
-                a[i] = Math.Cos(a[i]);
-              }
-              break;
+              if (postfixExpr[exprPos + 1] == 'o') {
+                // cos
+                exprPos += 4;
+                var a = stack[topOfStack];
+                for (int i = 0; i < N; i++) {
+                  a[i] = Math.Cos(a[i]);
+                }
+                break;
+              } else {
+                exprPos += 2;
+                // put 1 onto top of stack     // BUG!
+                topOfStack++;
+                var a = stack[topOfStack];
+                for (int i = 0; i < N; i++) a[i] = 1.0;
+                break;
+              }
             }
           case 's': {
@@ -241,4 +304,5 @@
       }
     }
+    */
     #endregion
   }
Index: /branches/2886_SymRegGrammarEnumeration/ExpressionClustering_R/ClusteringScript.R
===================================================================
--- /branches/2886_SymRegGrammarEnumeration/ExpressionClustering_R/ClusteringScript.R	(revision 15903)
+++ /branches/2886_SymRegGrammarEnumeration/ExpressionClustering_R/ClusteringScript.R	(revision 15903)
@@ -0,0 +1,13 @@
+library(largeVis)
+library(ggplot2)
+
+eval7 <- read.csv2("c:/Users/P24581/Desktop/evaluations_2018-04-11_12-50_TreeSize-7.csv",header = TRUE, sep = ";", dec=",")
+eval7_qs <- eval7[,3]
+eval7_x <- eval7[,5:105]
+lv <- largeVis(eval7_x, dim=2, K = 50, n_trees = 50) # TODO scale?
+clusters <- hdbscan(lv, minPts = 3, K=30)
+
+m <- data.frame(x=t(lv$coords)[,1], y=t(lv$coords)[,2], c=clusters$clusters, q=eval7_qs)
+
+
+ggplot(data=m, aes(x=x, y=y)) + geom_point(aes(color=q)) 
Index: /branches/2886_SymRegGrammarEnumeration/ExpressionClustering_R/ExpressionClustering_R.Rproj
===================================================================
--- /branches/2886_SymRegGrammarEnumeration/ExpressionClustering_R/ExpressionClustering_R.Rproj	(revision 15903)
+++ /branches/2886_SymRegGrammarEnumeration/ExpressionClustering_R/ExpressionClustering_R.Rproj	(revision 15903)
@@ -0,0 +1,13 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX
