source: branches/2886_SymRegGrammarEnumeration/ExpressionClustering/Program.cs @ 15840

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;

namespace ExpressionClustering {
  class Program {
    private static readonly string folder = Environment.GetFolderPath(Environment.SpecialFolder.DesktopDirectory);
    private static readonly string distinctSentencesFileName = Path.Combine(folder, @"distinctSentences.csv");
    private static readonly string allSentencesFileName = Path.Combine(folder, "allSentences.csv");
    private static readonly string outputFileName = Path.Combine(folder, "evaluations.csv.gz");
    private static readonly string columnDelimiter = ";";
    private static int N = 100;
    private static double[] evalBuf = new double[N];

    // pagie-1 (univariate)
    private static double min = -5.0;
    private static double max = +5.0;
    private static double[] xs = Enumerable.Range(1, N).Select(xi => ((double)xi / N) * (max - min) + min).ToArray(); // input
    private static double[] ys = xs.Select(xi => 1.0 / (1 + Math.Pow(xi, -4))).ToArray(); // target (necessary for scaling and clustering 


    public static int MAX_STACK = 20;
    public static double[][] stack = new double[MAX_STACK][];
    static Program() {
      for (int i = 0; i < MAX_STACK; i++)
        stack[i] = new double[N];
    }

    // loads symbolic expressions in postfix notation from a stream and identifies clusters of expressions
    static void Main(string[] args) {

      TestFLANN();

      var hash2Sentences = new Dictionary<string, List<string>>();



      // read all sentences and determine shortest sentences
      using (var reader = new StreamReader(allSentencesFileName)) {
        // read header
        reader.ReadLine();
        int nSentences = 0;
        while (!reader.EndOfStream) {
          var line = reader.ReadLine();
          var toks = line.Split(';');
          var hash = toks[2];
          List<string> ls;
          if (!hash2Sentences.TryGetValue(hash, out ls)) {
            ls = new List<string>(1);
            hash2Sentences.Add(hash, ls);
          }
          ls.Add(toks[1]);
          nSentences++;
        }

        Console.WriteLine("{0} {1}", nSentences, hash2Sentences.Count);
        //Evaluate(toks[1], xs, evalBuf);
      }

      using (var writer = new StreamWriter(new System.IO.Compression.GZipStream(new FileStream(outputFileName, FileMode.OpenOrCreate), System.IO.Compression.CompressionMode.Compress)))
        foreach (var kvp in hash2Sentences) {
          var ls = kvp.Value;
          var sentence = FindShortest(ls);
          Evaluate(sentence, xs, evalBuf);
          Add(writer, sentence, evalBuf);
        }
    }

    private static void TestFLANN() {
      var rand = new Random(1234);
      var dim = 100;
      var N = 10000;
      var dataset = new List<double[]>();
      for(int i=0;i<N;i++) {
        var x = new double[dim];
        for (int j = 0; j < dim; j++) x[j] = rand.NextDouble();
        dataset.Add(x);
      }
      List<int> nnResults;
      List<double> nnDists;
      Flann.FindNearestNeighbours(dataset, dataset, out nnResults, out nnDists);

    }

    private static string FindShortest(List<string> ls) {
      var minElem = ls.First();
      for (int i = 1; i < ls.Count; i++) {
        if (ls[i].Length < minElem.Length) minElem = ls[i];
      }
      return minElem;
    }


    #region evaluation
    // linear scaling
    private static void Scale(double[] evalBuf, double[] ys) {
      double alpha;
      double beta;
      HeuristicLab.Problems.DataAnalysis.OnlineCalculatorError error;

      HeuristicLab.Problems.DataAnalysis.OnlineLinearScalingParameterCalculator.Calculate(evalBuf, ys, out alpha, out beta, out error);
      if (error != HeuristicLab.Problems.DataAnalysis.OnlineCalculatorError.None) {
        alpha = 0.0;
        beta = 1.0;
        Console.WriteLine("WARNING: error in scaling");
      }

      for (int i = 0; i < evalBuf.Length; i++) {
        evalBuf[i] = beta * evalBuf[i] + alpha;
      }


      // 
      //   var meanE = 0.0;
      //   var meanE2 = 0.0;
      //   var meanY = 0.0;
      //   var meanY2 = 0.0;
      //   for(int i=0;i<evalBuf.Length;i++) {
      //     var deltaE = evalBuf[i] - meanE;
      //     meanE +=  deltaE / (i+1);
      //     var deltaE2 = evalBuf[i] - meanE;
      //     meanE2 += deltaE * deltaE2;
      // 
      // 
      //     var deltaY = ys[i] - meanY;
      //     meanY += deltaY / (i + 1);
      //     var deltaY2 = ys[i] - meanY;
      //     meanY2 += deltaY * deltaY2;
      //     TODO COVARIANCE
      //     Linear Scaling: b = cov(y,e) / var(e); a = meanY - b*meanE; 
      //   }
      // 
      //   var varE = meanE2 / evalBuf.Length;
      //   var varY = meanY2 / evalBuf.Length;
      //   Console.WriteLine("{0} {1} {2} {3}", meanE, evalBuf.Average(), meanY, ys.Average());
      //   Console.WriteLine("{0} {1}", varE, varY);
      // 
      //   var factor = varY / varE;
      //   for(int i=0;i<evalBuf.Length;i++) {
      //     evalBuf[i] = (evalBuf[i] - meanE) * factor + meanY;
      //   }
    }

    // evaluates postfix expressions (only for a very specific format)
    private static void Evaluate(string postfixExpr, double[] xs, double[] evalBuf) {
      int topOfStack = -1;
      Evaluate(postfixExpr, 0, xs, ref topOfStack);
      Array.Copy(stack[topOfStack], evalBuf, evalBuf.Length);
    }

    private static void Evaluate(string postfixExpr, int exprPos, double[] xs, ref int topOfStack) {
      while (exprPos < postfixExpr.Length) {
        switch (postfixExpr[exprPos]) {
          case '+': {
              exprPos += 2;
              var a = stack[topOfStack];
              var b = stack[topOfStack - 1];
              for (int i = 0; i < N; i++) {
                a[i] += b[i];
              }
              break;
            }
          case '*': {
              exprPos += 2;
              var a = stack[topOfStack];
              var b = stack[topOfStack - 1];
              for (int i = 0; i < N; i++) {
                a[i] *= b[i];
              }
              break;
            }
          case 'X': {
              exprPos += 2;
              topOfStack++;
              Array.Copy(xs, stack[topOfStack], N);
              break;
            }
          case 'c': {
              // cos
              exprPos += 4;
              var a = stack[topOfStack];
              for (int i = 0; i < N; i++) {
                a[i] = Math.Cos(a[i]);
              }
              break;
            }
          case 's': {
              // sin
              exprPos += 4;
              var a = stack[topOfStack];
              for (int i = 0; i < N; i++) {
                a[i] = Math.Sin(a[i]);
              }
              break;
            }
          case 'l': {
              // log
              exprPos += 4;
              var a = stack[topOfStack];
              for (int i = 0; i < N; i++) {
                a[i] = Math.Log(a[i]);
              }

              break;
            }
          case 'e': {
              // exp
              exprPos += 4;
              var a = stack[topOfStack];
              for (int i = 0; i < N; i++) {
                a[i] = Math.Exp(a[i]);
              }

              break;
            }
          case 'i': {
              // inv
              exprPos += 4;
              var a = stack[topOfStack];
              for (int i = 0; i < N; i++) {
                a[i] = 1.0 / a[i];
              }
              break;
            }
          default: {
              throw new InvalidOperationException(string.Format("Cannot handle {0} in {1}", postfixExpr[exprPos], postfixExpr));
            }
        }
      }
    }
    #endregion



    // add the line with it's evaluation result to a data structure for clustering
    private static void Add(StreamWriter writer, string line, double[] evalBuf) {
      var avg = evalBuf.Average();
      if (double.IsNaN(avg) || double.IsInfinity(avg)) {
        Console.WriteLine("skipping {0} {1}", evalBuf.Average(), line);
      } else {
        Scale(evalBuf, ys);

        writer.WriteLine(string.Join("\t", evalBuf.Select(ei => ei.ToString(System.Globalization.CultureInfo.InvariantCulture))));
      }
    }
  }
}