source: branches/2886_SymRegGrammarEnumeration/ExpressionClustering/Program.cs @ 15930

using System;
using System.Collections;
using System.Collections.Generic;
using System.Diagnostics;                                                                                              
using System.IO;
using System.Linq;                                                                                                     
using HeuristicLab.Problems.DataAnalysis;
using HeuristicLab.Problems.DataAnalysis.Symbolic;

// Reads sentences from files, determines the shortest infix expression for a hash and evaluates sentences on randomly generated data, evaluation result is written to file
namespace ExpressionClustering {
  class Program {
    //private static readonly string folder = @"D:\heal\documents\trunk\Publications\2018\GPTP\data";
    //private static readonly string clusterFolder = folder;
    //private static readonly string distinctSentencesFileName = Path.Combine(folder, @"distinctSentences_2018-04-13_16-40_TreeSize-7.csv.gz");
    //private static readonly string allSentencesFileName = Path.Combine(folder, "allSentences_2018-04-16_14-49_TreeSize-8_1d.csv.gz");
    //private static readonly string outputFileName = Path.Combine(folder, "evaluations_2018-04-16_14-49_TreeSize-8_1d.csv.gz");
    private static int N = 100;
    private static int PERF_STATS_UPDATE_INTERVAL = 100000;
    private static double[] evalBuf = new double[N];

    // pagie-1 (univariate)
    private static double min = -5.0;
    private static double max = +5.0;
    private static double[] xs = Enumerable.Range(1, N).Select(xi => ((double)xi / N) * (max - min) + min).ToArray(); // input

    private static double[] ys_pagie = xs.Select(xi => 1.0 / (1 + Math.Pow(xi, -4))).ToArray(); // a potential target (not used for search) 

    // x³  * exp(-x) * cos(x) * sin(x) * (sin(x)² * cos(x) - 1)
    // for keijzer x should be in scale 0 - 10 inclusive
    private static double[] ys_keijzer4 = xs
      .Select(xi => xi + 10.0) // scale 
      .Select(xi => xi * xi * xi + Math.Exp(-xi) * Math.Cos(xi) * Math.Sin(xi) * (Math.Sin(xi) * Math.Sin(xi) * Math.Cos(xi) - 1))
      .ToArray();

   
    // loads symbolic expressions in postfix notation from a stream and identifies clusters of expressions
    static void Main(string[] args) {
      var sentencesFileName = args[0];
      var outputFileName = Path.Combine(Path.GetDirectoryName(sentencesFileName), "evaluations_" + Path.GetFileName(sentencesFileName));
                                

      var hashToRowIdx = new Dictionary<string, int>();
      var hashToInfix = new Dictionary<string, string>();

      // read all sentences and determine shortest sentences
      using (var reader = new StreamReader(
        new System.IO.Compression.GZipStream(
          new FileStream(sentencesFileName, FileMode.Open, FileAccess.Read), 
          System.IO.Compression.CompressionMode.Decompress))) {
        // read header
        reader.ReadLine();
        int nSentences = 0;
        var sw = new Stopwatch();
        sw.Start();
        while (!reader.EndOfStream) {
          var line = reader.ReadLine();
          nSentences++;
          var toks = line.Split(';');
          var hash = toks[0];
          var length = toks[1];
          //var postfix = toks[2];
          var infix = toks[3];
          string expr;
          if (!hashToInfix.TryGetValue(hash, out expr)) {
            hashToInfix.Add(hash, infix);
            hashToRowIdx.Add(hash, nSentences);
          }
          else if(expr.Length > infix.Length) {
            hashToInfix[hash] = infix;  // keep only shortest
            hashToRowIdx[hash] = nSentences;
          }
          if (nSentences % PERF_STATS_UPDATE_INTERVAL == PERF_STATS_UPDATE_INTERVAL-1) {
            Console.WriteLine("Read perf: {0} sentences in {1}ms", PERF_STATS_UPDATE_INTERVAL, sw.ElapsedMilliseconds);
            sw.Restart();
          }
        }

        Console.WriteLine("{0} {1}", nSentences, hashToInfix.Count);
        //Evaluate(toks[1], xs, evalBuf);
      }

      Scale(ys_keijzer4);
      Scale(ys_pagie);

      // output all functions
      using (var writer = new StreamWriter(
        new System.IO.Compression.GZipStream(
          new FileStream(outputFileName, FileMode.OpenOrCreate),
          System.IO.Compression.CompressionMode.Compress))) {
        var sw = new Stopwatch();
        sw.Start();

        var ds = new Dataset(new string[] { "X" }, new IList[] { xs });
        writer.WriteLine("{0};{1};{2};{3};{4};{5}", "Hash", "RowIdx (in allSentences)", "NMSE pagie", "NMSE keijzer4", "infix", 
          string.Join(";", Enumerable.Range(0, xs.Length).Select(i => "eval" + i)));
        int nSentences = 0;
        foreach (var kvp in hashToInfix) {
          var hash = kvp.Key;
          var infixExpr = kvp.Value;
          evalBuf = EvaluateInfix(infixExpr, ds).ToArray();
          if (evalBuf.Any(ei => double.IsInfinity(ei) || double.IsNaN(ei))) {
            //Console.WriteLine("skipping {0} {1}", evalBuf.Average(), infixExpr);
            //Console.Write(".");
          } else {
            try {
              Scale(evalBuf);
              // functions.Add((double[])evalBuf.Clone());
              // sentences.Add(sentence);
              OnlineCalculatorError error;
              var nmse_pagie = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(evalBuf, ys_pagie, out error);
              if (error != OnlineCalculatorError.None) nmse_pagie = 10;
              var nmse_keijzer = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(evalBuf, ys_keijzer4, out error);
              if (error != OnlineCalculatorError.None) nmse_keijzer = 10;
              writer.WriteLine("{0};{1};{2};{3};{4};{5}", hash,  hashToRowIdx[hash], nmse_pagie, nmse_keijzer, infixExpr, 
                string.Join(";", evalBuf.Select(fi => fi.ToString())));
            } catch (ArgumentException e) {
              // scaling failed
            }
          }
          
          if (nSentences++ % PERF_STATS_UPDATE_INTERVAL == PERF_STATS_UPDATE_INTERVAL-1) {
            Console.WriteLine("Eval perf: {0} sentences in {1}ms expected time remaining: {2}min", 
              PERF_STATS_UPDATE_INTERVAL, sw.ElapsedMilliseconds,
              (hashToRowIdx.Count - nSentences) / (double)PERF_STATS_UPDATE_INTERVAL * sw.ElapsedMilliseconds / 1000 / 60);
            sw.Restart();
          }
        }
      }
    }


    #region evaluation

    // scaling to zero-mean unit variance  (ignore NaN and +/-Inf.
    private static void Scale(double[] evalBuf) {
      double mean;
      double variance;
      var max = evalBuf.Select(xi=>Math.Abs(xi)).Max();
      for (int i = 0; i < evalBuf.Length; i++) {
        evalBuf[i] /= max;
      }

      OnlineCalculatorError error, varError;
      OnlineMeanAndVarianceCalculator.Calculate(evalBuf.Where(xi => !double.IsNaN(xi) && !double.IsInfinity(xi)), out mean, out variance, out error, out varError);
      if(error!=OnlineCalculatorError.None || varError != OnlineCalculatorError.None) {
        throw new ArgumentException("Cannot scale vector");
      }

      for (int i = 0; i < evalBuf.Length; i++) {
        if (double.IsNaN(evalBuf[i])) evalBuf[i] = mean;
        else if (double.IsPositiveInfinity(evalBuf[i])) evalBuf[i] = 10;
        else if (double.IsNegativeInfinity(evalBuf[i])) evalBuf[i] = -10.0;
        evalBuf[i] = 1.0 / variance * evalBuf[i] + mean;
      }
    }

    // linear scaling to match target
    private static void Scale(double[] evalBuf, double[] ys) {
      double alpha;
      double beta;
      HeuristicLab.Problems.DataAnalysis.OnlineCalculatorError error;

      HeuristicLab.Problems.DataAnalysis.OnlineLinearScalingParameterCalculator.Calculate(evalBuf, ys, out alpha, out beta, out error);
      if (error != HeuristicLab.Problems.DataAnalysis.OnlineCalculatorError.None) {
        throw new ArgumentException();
      }

      for (int i = 0; i < evalBuf.Length; i++) {
        evalBuf[i] = beta * evalBuf[i] + alpha;
      }
    }

    // evaluates infix expressions (using the infix parser)
    private static IEnumerable<double> EvaluateInfix(string infixExpr, Dataset ds) {
      var parser = new HeuristicLab.Problems.DataAnalysis.Symbolic.InfixExpressionParser();
      var tree = parser.Parse(infixExpr);
      var interpreter = new SymbolicDataAnalysisExpressionTreeLinearInterpreter();
      return interpreter.GetSymbolicExpressionTreeValues(tree, ds, Enumerable.Range(0, ds.Rows));
    }         
    #endregion
  }
}