source: branches/1888_OaaS/HeuristicLab.Services.Optimization.Controller/HL/HiveScenarioManager.cs @ 17869

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HeuristicLab.Services.Optimization.ControllerService.Interfaces;
using HeuristicLab.Optimization;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using HeuristicLab.Problems.TravelingSalesman;
using HeuristicLab;
using System.Reflection;
using HeuristicLab.Services.Optimization.ControllerService.Model;
using HeuristicLab.Core;
using System.Collections;
using HeuristicLab.Clients.Hive;
using System.Threading;
using HeuristicLab.Data;
using System.IO;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.StorageClient;
using HeuristicLab.Services.Optimization.ControllerService.Azure;
using System.Data;
using HeuristicLab.Services.Optimization.ControllerService.General;
using HeuristicLab.Services.Optimization.ControllerService.Parameters.HL;

namespace HeuristicLab.Services.Optimization.ControllerService {
  public class ScenarioEntity : TableServiceEntity {    
    public ScenarioEntity() {
    }
    
    public ScenarioEntity(string scenarioName, string scenario, string mapper) {
      PartitionKey = "ScenarioPartition";
      RowKey = scenarioName;
      Scenario = scenario;
      Mapper = mapper;
    }    

    public string Scenario { get; set; }

    public string Mapper { get; set; }
  }

  public class HiveScenarioManager : IScenarioManager {
    private static HLParameterMapper parameterMapper = new HLParameterMapper();
    private static IScenarioMapper tspMapper;
    private static object lockable;
    private Dictionary<string, IScenarioMapper> mappers = new Dictionary<string, IScenarioMapper>();
    private IDataAccessLayer dal = DataAccessLayerProvider.GetLayer();

    public static readonly string SCENARIO_TABLE = "Scenario";

    static HiveScenarioManager() {
      lockable = new object();
    }

    public string DispatchScenario(Model.User user, Model.OptimizationScenario scenario, JobExecutionDetails details) {
      // Experiment experiment = new Experiment();
      // var problem = new TravelingSalesmanProblem();
      // var algo = new GeneticAlgorithm();
      // algo.Problem = problem;      
      
      IScenarioMapper mapper = GetMapper(scenario.Id);
      IAlgorithm algo;
      mapper.MapScenario(scenario, out algo);
      /*if (details.Repititions > 1) {
        BatchRun br = new BatchRun();
        //br.Name = details.JobTitle;
        br.Optimizer = algo;        
        br.Repetitions = details.Repititions;
        return SendExperimentToHive(user, br, details);
      }
      else {       
        
      }*/
      return SendExperimentToHive(user, algo, details);
    }

    private sealed class StackEntry {
      public HeuristicLab.Optimization.Experiment Parent { get; set; }
      public IList<Model.Algorithm> Children { get; set; }
    }

    public bool DispatchExperiment(User user, Model.Experiment exp, JobExecutionDetails details) {
      // For now the experiment will be flatened for execution
      HeuristicLab.Optimization.Experiment hiveExperiment = new HeuristicLab.Optimization.Experiment(exp.Name);
      var stack = new Stack<StackEntry>();
      var children = new List<Model.Algorithm>();
      foreach (var child in exp.Algorithm) {
        children.Add(child);        
      }
      stack.Push(new StackEntry() { Parent = hiveExperiment, Children = children });

      while (stack.Count > 0) {
        var entry = stack.Pop();
        // handle current entry
        foreach (var child in entry.Children) {
          // This is a template experiment; 
          if (child.IsExperiment) {
            var parent = new HeuristicLab.Optimization.Experiment();
            entry.Parent.Optimizers.Add(parent);
            stack.Push(new StackEntry() { Parent = parent, Children = child.ChildAlgorithms });          
          }
          // This entity is mapable
          else {           
            IScenarioMapper mapper = GetMapper(child.Id);
            var optScen = new OptimizationScenario() { Id = child.Id };
            optScen.Algorithm.Add(child);
            IAlgorithm algo;
            mapper.MapScenario(optScen, out algo);
            algo.Name = child.Id;
            entry.Parent.Optimizers.Add(algo);
          }  
        }
      }
      details.JobTitle = exp.Name;
      var jobId = SendExperimentToHive(user, hiveExperiment, details);

      // add relationship to azure tablestorage      
      if (jobId != null) {
        dal.JobDao.Add(user.Username, exp, jobId);
      }
      return jobId != null;
    }

    static public string AssemblyDirectory {
      get {
        string codeBase = Assembly.GetExecutingAssembly().CodeBase;
        UriBuilder uri = new UriBuilder(codeBase);
        string path = Uri.UnescapeDataString(uri.Path);
        return Path.GetDirectoryName(path);
      }
    }

    public static IScenarioMapper CompileMapper(string scenarioMapper) {
       // http://stackoverflow.com/questions/3188882/compile-and-run-dynamic-code-without-generating-exe
      using (var csCodeProvider = new Microsoft.CSharp.CSharpCodeProvider()) {
        var cp = new System.CodeDom.Compiler.CompilerParameters() {
          GenerateInMemory = true
        };

        var referencedPaths = Directory.GetFiles(AssemblyDirectory, "*.dll");
        foreach (var loadedAssembly in referencedPaths) {
          cp.ReferencedAssemblies.Add(loadedAssembly);
        }
        cp.ReferencedAssemblies.Add("System.dll");
        cp.ReferencedAssemblies.Add("System.Core.dll");
        cp.ReferencedAssemblies.Add("System.Data.dll");
        cp.ReferencedAssemblies.Add("System.Xml.dll");
        cp.ReferencedAssemblies.Add("System.Xml.Linq.dll");

        var res = csCodeProvider.CompileAssemblyFromSource(
          cp,
          scenarioMapper
        );

        foreach (var error in res.Errors) {
          Console.WriteLine(error);
        }
        
        var firstMapper = res.CompiledAssembly.GetTypes().Where(p => typeof(IScenarioMapper).IsAssignableFrom(p)).FirstOrDefault();        
        return Activator.CreateInstance(firstMapper) as IScenarioMapper;
      }
    }

    private IScenarioMapper GetMapper(string scenarioId) {
      var id = scenarioId;
      IScenarioMapper mapper = null;      
      if (!mappers.TryGetValue(id, out mapper)) {
        lock (lockable) {
          if (mappers.ContainsKey(id))
            return mappers[id];
          
          var mapperString = GetMapperFromBlobStore(id);
          if (mapperString == null) return null;
          mapper = CompileMapper(mapperString);

          mappers[id] = mapper;
          return mapper;
        } // lock        
       } // if
      return mapper;
     }  

    private void MapExperiment(Model.OptimizationScenario scenario, out IAlgorithm algorithm) {      
      IScenarioMapper mapper = GetMapper(scenario.Id);
      mapper.MapScenario(scenario, out algorithm);     
    }

    private HiveServiceLocator ConfigureHive(Model.User user) {
      var serviceLocator = new HiveServiceLocator();
      serviceLocator.Username = user.Username;
      serviceLocator.Password = user.Password;
      serviceLocator.EndpointConfigurationName = Configuration.HiveEndpointName;
      return serviceLocator;
    }

    private string SendExperimentToHive(Model.User user, IOptimizer exp, JobExecutionDetails details) {      
      var job = new RefreshableJob();
      job.IsAllowedPrivileged = true;      
      job.Job.Name = details.JobTitle;
      job.Job.ResourceNames = details.Group;      
      job.RefreshAutomatically = false;
      
      if (details.Repititions > 1) {
        BatchRun br = new BatchRun();
        br.Optimizer = exp;
        br.Repetitions = details.Repititions;
        exp = br;
      }

      if (exp.ExecutionState != ExecutionState.Prepared) {
        exp.Prepare();
      }      
      job.HiveTasks.Add(new OptimizerHiveTask(exp));
            
      var locator = ConfigureHive(user);
      HiveClient.Store(job, new CancellationToken(), locator);
      
      job.StopResultPolling();
      return job.Id != null ? job.Id.ToString() : null;
    }


    public IList<Model.Job> GetJobs(User user) {
      var serviceLocator = ConfigureHive(user);
      var jobsLoaded = serviceLocator.CallHiveService<IEnumerable<HeuristicLab.Clients.Hive.Job>>(s => s.GetJobs());
      IList<Model.Job> jobs = new List<Model.Job>();

      foreach (var job in jobsLoaded) {
        jobs.Add(ConvertJob(user, job));
      }      
      return jobs;
    }

    private Model.Job ConvertJob(User user, HeuristicLab.Clients.Hive.Job job) 
    {
      var waitingJobs = job.JobCount - job.CalculatingCount - job.FinishedCount;
      Model.JobState? state = null;
      if (job.CalculatingCount > 0)
        state = JobState.Calculating;
      else if (job.JobCount > 0 && job.JobCount == job.FinishedCount)
        state = JobState.Finished;
      else
        state = JobState.Waiting;

      return new Model.Job() { Id = job.Id.ToString(), Name = job.Name, Resource = job.ResourceNames, State = state.Value, DateCreated = job.DateCreated };
    }


    public Model.Job GetJob(User user, string id) {
      var serviceLocator = ConfigureHive(user);
      var guid = Guid.Parse(id);
      return ConvertJob(user, serviceLocator.CallHiveService<HeuristicLab.Clients.Hive.Job>(s => s.GetJob(guid)));
    }


    public bool DeleteJob(User user, string id) {
      var serviceLocator = ConfigureHive(user);
      var guid = Guid.Parse(id);      
      serviceLocator.CallHiveService(s => s.DeleteJob(guid));
      return true;
    }

    public IList<Model.Run> GetJobResults(User user, string id) {
      var serviceLocator = ConfigureHive(user);
      var guid = Guid.Parse(id);
      var jobTasks = serviceLocator.CallHiveService<IEnumerable<HeuristicLab.Clients.Hive.LightweightTask>>(s => s.GetLightweightJobTasks(guid));

      IList<Guid> taskIds = new List<Guid>();
      foreach (var task in jobTasks) {
        taskIds.Add(task.Id);
      }
 
      TaskDownloader downloader = new TaskDownloader(taskIds, ConfigureHive(user));
      downloader.StartAsync();
      while (!downloader.IsFinished) {        
        Thread.Sleep(500);

        if (downloader.IsFaulted) {
          throw downloader.Exception;
        }
      }

      IDictionary<Guid, HiveTask> hiveTasks = downloader.Results;
      IList<Model.Run> runs = new List<Model.Run>();
      foreach (var keyTask in hiveTasks.Keys) {
        var oht = hiveTasks[keyTask] as OptimizerHiveTask;
        if (oht != null) {
          foreach (var run in oht.ItemTask.Item.Runs) {
            Model.Run taskRun = new Model.Run();
            taskRun.Id = taskRun.Name = run.Name;
            // We try to extract the original algorithm name here
            var index = taskRun.Name.LastIndexOf(" Run ");
            var algorithName = taskRun.Name.Substring(0, index);
            taskRun.AlgorithmName = algorithName;
            if (taskRun.AlgorithmName == null)
              taskRun.AlgorithmName = run.Algorithm != null ? run.Algorithm.Name : run.Name;
            
            IList<Parameter> resultValues = new List<Model.Parameter>();
            foreach (var key in run.Results.Keys) {
              var value = run.Results[key];              
              Parameter result = MapHiveDataType(key, value);
              resultValues.Add(result);              
            }
            taskRun.Results = resultValues;
            IList<Parameter> inputParameters = new List<Model.Parameter>();
            foreach (var key in run.Parameters.Keys) {
              var value = run.Parameters[key];
              Parameter param = MapHiveDataType(key, value);
              inputParameters.Add(param);
            }
            // crawl the copied experiment of the job
            //taskRun.Experiment = dal.JobDao.FindByJobId(id);
            taskRun.InputParameters = inputParameters;
            runs.Add(taskRun);
          }
        }
      }
      return runs;
    }
    
    private Parameter MapHiveDataType(string name, IItem item) {

      if (parameterMapper.IsHandlerAvailable(item)) {
        return parameterMapper.Map(name, item);
      }

      var result = new Parameter();
      result.Type = ParameterType.String;
      if (item is IStringConvertibleValue) {
        var value = (item as IStringConvertibleValue).GetValue();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value };
      }
      else if (item is IStringConvertibleValueTuple) {
        var value1 = (item as IStringConvertibleValueTuple).Item1.GetValue();
        var value2 = (item as IStringConvertibleValueTuple).Item2.GetValue();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = "{" + value1 + ", " + value2 + "}" };
      }           
      else if (item is DoubleArray) {
        var array = item as DoubleArray;
        double[] arrayValue = new double[array.Length];
        for (int i = 0; i < arrayValue.Length; ++i) {
          arrayValue[i] = array[i];
        }
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.DecimalVector() { Name = name, Value = arrayValue };
      }
      else if (item is DoubleMatrix) {
        var matrix = item as DoubleMatrix;
        /*double[][] matrixValue = new double[matrix.Rows][];
        for (int i = 0; i < matrixValue.Length; ++i) {
          matrixValue[i] = new double[matrix.Columns];
          for (int j = 0; j < matrixValue[i].Length; ++j) {
            matrixValue[i][j] = matrix[i, j];
          }
        } */
        double[][] matrixValue = new double[matrix.Columns][];
        for (int i = 0; i < matrix.Columns; ++i) {
          matrixValue[i] = new double[matrix.Rows];
          for (int j = 0; j < matrix.Rows; ++j) {
            matrixValue[i][j] = matrix[j, i];
          }
        }
        matrixValue = transpose(matrixValue);
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.DecimalMatrix() { Name = name, Value = matrixValue, RowNames = (matrix.ColumnNames.Count() > 0 ? matrix.ColumnNames.ToArray() : null) };
      }
      else if (item is IStringConvertibleArray) {
        StringBuilder sb = new StringBuilder();
        var array = item as IStringConvertibleArray;
        if (array.Length == 0) {
          sb.Append("[ ]");
        }
        else {
          sb.Append("[");
          for (int i = 0; i < array.Length - 1; i++) {
            sb.Append(array.GetValue(i)).Append(", ");
          }
          sb.Append(array.GetValue(array.Length - 1));
          sb.Append(" ]");
        }
        var value = sb.ToString();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value };
      }
      else if (item is HeuristicLab.Analysis.DataTable) {
        var table = item as HeuristicLab.Analysis.DataTable;
        string[] names = new string[table.Rows.Count];
        double[][] results = new double[table.Rows.Count][];
        for (int i = 0; i < table.Rows.Count; i++ ) {
          var columns = table.Rows.ToList()[i];
          names[i] = columns.Name;
          results[i] = new double[columns.Values.Count];          
          for (int j=0; j < columns.Values.Count; j++) {
            results[i][j] = columns.Values[j];
          }
        }
        results = transpose(results);
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.DecimalMatrix() { Name = name, Value = results, RowNames = names };
      }
      else if (item is IStringConvertibleMatrix) {
        StringBuilder sb = new StringBuilder();
        var matrix = item as IStringConvertibleMatrix;

        if (matrix.Rows == 0 || matrix.Columns == 0) {
          sb.Append("[ ]");
        }
        else {

          sb.Append("[ ");
          for (int r = 0; r < matrix.Rows; r++) {
            sb.Append("( ");
            for (int c = 0; c < matrix.Columns - 1; c++) {
              matrix.GetValue(r, c);
              sb.Append(matrix.GetValue(r, c)).Append(", ");
            }
            sb.Append(matrix.GetValue(r, matrix.Columns - 1)).Append(r < matrix.Rows - 1 ? " ), " : " )");
          }
          sb.Append(" ]");
        }
        var value = sb.ToString();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value };
      }
      else {
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = item.ItemName != null ? item.ItemName + " (Cannot be displayed properly as string)" : "Cannot be displayed properly as string" };
      }
      return result;
    }

    public bool AddScenario(User user, string scenarioName, string scenarioXml, string scenarioMapper) {
      // create scenario mapper
      var mapper = CompileMapper(scenarioMapper);
      if (mapper == null)
        return false;
      
      // insert into table & blob store
      var scenDao = dal.ScenarioDao;
      var blobDao = dal.BlobDao;
      
      Guid scenarioXmlGuid = Guid.NewGuid();
      Guid scenarioMapperGuid = Guid.NewGuid();
      string scenarioXmlId = scenarioName + "_" + scenarioXmlGuid.ToString();
      string scenarioMapperId = scenarioName + "_" + scenarioMapperGuid.ToString();
      if (!blobDao.Add(new StringEntry() { Key = scenarioXmlId, Text = scenarioXml }))
        return false;
      if (!blobDao.Add(new StringEntry() { Key = scenarioMapperId, Text = scenarioMapper }))
        return false;
      if (!scenDao.Add(new ScenarioEntity(scenarioName, scenarioXmlId, scenarioMapperId)))
        return false;

      // everything stored in the DB -> add mapper to dictionary
      if (!mappers.ContainsKey(scenarioName)) {
        lock (lockable) {
          if (!mappers.ContainsKey(scenarioName))
            mappers[scenarioName] = mapper;
        }
      }
      return true;
    }

    private double[][] transpose(double[][] arr) {
      var width = arr.Length;
      var height = arr[0].Length;

      var res = new double[height][];
      for (var i = 0; i < height; i++) 
        res[i] = new double[width];

      for (var i = 0; i < width; i++) {
        for (var j = 0; j < height; j++) {
          res[j][i] = arr[i][j];
        }
      }
      return res;
    }

    private string GetMapperFromBlobStore(string scenarioName) {
      var scenarioDao = dal.ScenarioDao;
      var blobDao = dal.BlobDao;
      var entity = scenarioDao.FindByName(scenarioName);
      if (entity == null)
        return null;

      var mapper = blobDao.FindByKey(entity.Mapper);
      if (mapper == null)
        return null;
      return mapper.Text;
    }

    public bool DeleteScenario(User user, string scenarioName) {
      // delete from table & blob store
      var scenarioDao = dal.ScenarioDao;
      var blobDao = dal.BlobDao;

      var entity = scenarioDao.FindByName(scenarioName);
      if (entity == null)
        return false;

      blobDao.DeleteByKey(entity.Mapper);
      blobDao.DeleteByKey(entity.Scenario);
      scenarioDao.DeleteByName(scenarioName);            
      return true;
    }

    public string SaveExperiment(User user, Model.Experiment experiment) {
      if (experiment.Id != null) {
        dal.ExperimentDao.Update(user.Username, experiment);
      }
      else {
        dal.ExperimentDao.Add(user.Username, experiment);
      }
      return experiment.Id;
    }

    public IEnumerable<string> GetExperimentNames(User user) {
      return (from exp in dal.ExperimentDao.GetExperiments(user.Username)
              select exp.Name);
    }


    public bool DeleteExperiment(User user, string experiment) {
      return dal.ExperimentDao.Delete(user.Username, experiment);
    }


    public Model.Task GetTaskData(User u, string jobId, string taskId) {
      TaskDownloader downloader = new TaskDownloader(new List<Guid>(){Guid.Parse(taskId)}, ConfigureHive(u));
      downloader.StartAsync();
      while (!downloader.IsFinished) {
        Thread.Sleep(250);
        if (downloader.IsFaulted) {
          throw downloader.Exception;
        }
      }
      
      IDictionary<Guid, HiveTask> hiveTasks = downloader.Results;
      var task = hiveTasks[Guid.Parse(taskId)];
      if (task == null) 
        return null;
      
      return new Model.Task() {
        State = new Model.TaskState() {
          DateCreated = task.Task.DateCreated.ToString(),
          DateFinished = task.Task.DateFinished.ToString(),
          ExecutionTime = task.Task.ExecutionTime.ToString(),
          State = task.Task.State.ToString()
        },
        General = new Model.General() {
          Id = task.Task.Id.ToString(),
          LastChanged = task.Task.Modified.ToString(),
          Name = task.ItemTask.Name
        }
      };
    }

    public Model.Job GetTasks(User u, string jobId) {
      var serviceLocator = ConfigureHive(u);
      var jobTasks = serviceLocator.CallHiveService<IEnumerable<HeuristicLab.Clients.Hive.LightweightTask>>(s => s.GetLightweightJobTasks(Guid.Parse(jobId)));
      
      var job = new Model.Job();
      job.Id = jobId;
      var tasks = new Dictionary<Guid, Model.Task>();
      // push all elements to dictionary
      foreach (var task in jobTasks) {
        var children = serviceLocator.CallHiveService<IEnumerable<HeuristicLab.Clients.Hive.LightweightTask>>(s => s.GetLightweightChildTasks(Guid.Parse(jobId), true, true));
        foreach (var child in children) {
          tasks[child.Id] = new Model.Task() {
            State = new Model.TaskState() {
              DateCreated = child.DateCreated.ToString(),
              DateFinished = child.DateFinished.ToString(),
              ExecutionTime = child.ExecutionTime.ToString(),
              State = child.State.ToString()
            },
            General = new Model.General() {
              Id = child.Id.ToString(),
              LastChanged = child.Modified.ToString(),
              Name = child.ItemName
            }
          };
        }
      }

      // traverse all tasks again and create tree of tasks
      foreach (var task in jobTasks) {
        if (task.ParentTaskId.HasValue)
          tasks[task.Id].Children.Add(tasks[task.ParentTaskId.Value]);
        else // its a root task
          job.Tasks.Add(tasks[task.Id]);
      }

      return job;
    }


    public Model.Experiment GetExperimentByName(User user, string scenario) {
      return dal.ExperimentDao.GetExperimentByName(user.Username, scenario);
    }

    public IEnumerable<Model.Experiment> GetExperiments(User user, bool namesOnly=false) {
      return dal.ExperimentDao.GetExperiments(user.Username, namesOnly);
    }

    public Model.Experiment GetExperimentById(User user, string nodeId) {
      return dal.ExperimentDao.GetExperimentById(user, nodeId);
    }


    public string GetVisualExtension(string algorithmId) {
      return dal.VisualExtensionDao.FindById(algorithmId);
    }


    public bool AddVisualExtension(string algorithmId, string script) {
      return dal.VisualExtensionDao.Add(algorithmId, script);
    }

    public bool DeleteVisualExtension(string algorithmId) {
      return dal.VisualExtensionDao.DeleteById(algorithmId);
    }


    public bool ExistsVisualExtension(string algorithmId) {
      return dal.VisualExtensionDao.Exists(algorithmId);
    }
  }
}