source: branches/OaaS/HeuristicLab.Services.Optimization.Controller/HiveScenarioManager.cs @ 8916

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HeuristicLab.Services.Optimization.ControllerService.Interfaces;
using HeuristicLab.Optimization;
using HeuristicLab.Algorithms.GeneticAlgorithm;
using HeuristicLab.Problems.TravelingSalesman;
using HeuristicLab;
using System.Reflection;
using HeuristicLab.Services.Optimization.ControllerService.Model;
using HeuristicLab.Core;
using System.Collections;
using HeuristicLab.Clients.Hive;
using System.Threading;
using HeuristicLab.Data;
using System.IO;

namespace HeuristicLab.Services.Optimization.ControllerService {
  public class HiveScenarioManager : IScenarioManager {
    private static IScenarioMapper tspMapper;
    private static object lockable;

    static HiveScenarioManager() {
      lockable = new object();
    }

    public void DispatchScenario(Model.User user, Model.OptimizationScenario scenario) {
      Experiment experiment = new Experiment();
      var problem = new TravelingSalesmanProblem();
      var algo = new GeneticAlgorithm();
      algo.Problem = problem;      
      
      MapExperiment(scenario, algo);

      experiment.Optimizers.Add(algo);
      SendExperimentToHive(user, experiment);
    }

    static public string AssemblyDirectory {
      get {
        string codeBase = Assembly.GetExecutingAssembly().CodeBase;
        UriBuilder uri = new UriBuilder(codeBase);
        string path = Uri.UnescapeDataString(uri.Path);
        return Path.GetDirectoryName(path);
      }
    }

    private IScenarioMapper GetMapper(Model.OptimizationScenario scenario) {
      if (tspMapper == null) {
        lock (lockable) {
          if (tspMapper != null)
            return tspMapper;
          
          // http://stackoverflow.com/questions/3188882/compile-and-run-dynamic-code-without-generating-exe
          using (var csCodeProvider = new Microsoft.CSharp.CSharpCodeProvider()) {
            var cp = new System.CodeDom.Compiler.CompilerParameters() {
              GenerateInMemory = true
            };

            var referencedPaths = Directory.GetFiles(AssemblyDirectory, "*.dll");
            foreach (var loadedAssembly in referencedPaths) {
              cp.ReferencedAssemblies.Add(loadedAssembly);
            }
            cp.ReferencedAssemblies.Add("System.dll");
            cp.ReferencedAssemblies.Add("System.Core.dll");
            cp.ReferencedAssemblies.Add("System.Data.dll");
            cp.ReferencedAssemblies.Add("System.Xml.dll");
            cp.ReferencedAssemblies.Add("System.Xml.Linq.dll");

            var res = csCodeProvider.CompileAssemblyFromSource(
              cp,
            @"using System;
              using System.Collections.Generic;
              using System.Linq;
              using System.Text;
              using HeuristicLab.Data;
              using HeuristicLab.Services.Optimization.ControllerService.Model; 
              using HeuristicLab.Optimization;
              using HeuristicLab.Services.Optimization.ControllerService;
              using HeuristicLab.Algorithms.GeneticAlgorithm;
              using HeuristicLab.Problems.TravelingSalesman;
              public class TSPScenarioMapper : IScenarioMapper {
                public void MapScenario(OptimizationScenario scenario, IAlgorithm algorithm) {
                  Console.WriteLine(""Mapping scenario!"");
                  var ga = algorithm as GeneticAlgorithm;
                  var problem = algorithm.Problem as TravelingSalesmanProblem;
                  
                  problem.BestKnownQuality = HLMapper.ConvertToDoubleValue(scenario.InputParameters.FindByName(""BestKnownQuality""));
                  problem.BestKnownSolution = HLMapper.ConvertToPermutation(scenario.InputParameters.FindByName(""BestKnownSolution""));
                  problem.Coordinates = HLMapper.ConvertToDoubleMatrix(scenario.InputParameters.FindByName(""Coordinates""));
                  var evalParam = HLMapper.GetStringValue(scenario.InputParameters.FindByName(""EvaluatorParameter""));
                  if (evalParam == ""HeuristicLab.Problems.TravelingSalesman.TSPRoundedEuclideanPathEvaluator"") {
                    problem.EvaluatorParameter.Value = new TSPRoundedEuclideanPathEvaluator();
                  }
                  else if (evalParam == ""HeuristicLab.Problems.TravelingSalesman.TSPGeoPathEvaluator"") {
                    problem.EvaluatorParameter.Value = new TSPGeoPathEvaluator();
                  }
                  else {
                    problem.EvaluatorParameter.Value = new TSPEuclideanPathEvaluator();
                  }
                  problem.UseDistanceMatrix = HLMapper.ConvertToBoolValue(scenario.InputParameters.FindByName(""UseDistanceMatrix""));
                  Console.WriteLine(""Mapping algorithm..."");
                  ga.Mutator = HLMapper.FindInItemSet<HeuristicLab.Optimization.IManipulator>(ga.MutatorParameter.ValidValues, scenario.AlgorithmParameters.FindByName(""Mutator""));
                  ga.CrossoverParameter.Value = HLMapper.FindOperator<HeuristicLab.Optimization.ICrossover>(problem, scenario.AlgorithmParameters.FindByName(""CrossoverParameter""));
                  ga.Elites = HLMapper.ConvertToIntValue(scenario.AlgorithmParameters.FindByName(""Elites""));
                  ga.MaximumGenerations = HLMapper.ConvertToIntValue(scenario.AlgorithmParameters.FindByName(""MaximumGenerations""));
                  ga.MutationProbability = HLMapper.ConvertToPercentValue(scenario.AlgorithmParameters.FindByName(""MutationProbability""));                
                  ga.PopulationSize = HLMapper.ConvertToIntValue(scenario.AlgorithmParameters.FindByName(""PopulationSize""));
                  ga.Seed = HLMapper.ConvertToIntValue(scenario.AlgorithmParameters.FindByName(""Seed""));
                  ga.Selector = HLMapper.FindInItemSet<HeuristicLab.Optimization.ISelector>(ga.SelectorParameter.ValidValues, scenario.AlgorithmParameters.FindByName(""Selector""));
                  ga.SetSeedRandomly = HLMapper.ConvertToBoolValue(scenario.AlgorithmParameters.FindByName(""SetSeedRandomly""));
                }
              }"
            );

            foreach (var error in res.Errors) {
              Console.WriteLine(error);
            }

            var type = res.CompiledAssembly.GetType("TSPScenarioMapper");
            tspMapper = Activator.CreateInstance(type) as IScenarioMapper;
          } // using
        } // lock        
       } // if
       return tspMapper;
     }  

    private void MapExperiment(Model.OptimizationScenario scenario, IAlgorithm algorithm) {      
      IScenarioMapper mapper = GetMapper(scenario);
      mapper.MapScenario(scenario, algorithm);     
    }

    private void SendExperimentToHive(Model.User user, Experiment exp) {
      var job = new RefreshableJob();
      job.IsAllowedPrivileged = true;      
      job.Job.Name = "Web Scheduled Traveling Salesman Job";
      job.Job.ResourceNames = "TESTGROUP";
      job.RefreshAutomatically = false;
      job.HiveTasks.Add(new OptimizerHiveTask(exp));
      HiveServiceLocator.Instance.Username = user.Username;
      HiveServiceLocator.Instance.Password = user.Password;
      HiveServiceLocator.Instance.EndpointConfigurationName = "WSHttpBinding_IHiveService";
      
      HiveClient.StartJob((ex) => { 
        Console.WriteLine(ex.StackTrace); 
      }, job, new CancellationToken());

      job.StopResultPolling();
    }


    public IList<Model.Job> GetJobs(User user) {
      HiveServiceLocator.Instance.Username = user.Username;
      HiveServiceLocator.Instance.Password = user.Password;
      HiveServiceLocator.Instance.EndpointConfigurationName = "WSHttpBinding_IHiveService";
      var jobsLoaded = HiveServiceLocator.Instance.CallHiveService<IEnumerable<HeuristicLab.Clients.Hive.Job>>(s => s.GetJobs());
      IList<Model.Job> jobs = new List<Model.Job>();
      
      foreach (var job in jobsLoaded) {
        jobs.Add(ConvertJob(user, job));
      }
      return jobs;
    }

    private Model.Job ConvertJob(User user, HeuristicLab.Clients.Hive.Job job) 
    {
      var waitingJobs = job.JobCount - job.CalculatingCount - job.FinishedCount;
      Model.JobState? state = null;
      var jobTasks = HiveServiceLocator.Instance.CallHiveService<IEnumerable<HeuristicLab.Clients.Hive.LightweightTask>>(s => s.GetLightweightJobTasks(job.Id));
      
      foreach (var task in jobTasks) {        
        switch (task.State) {
          case TaskState.Aborted:
            state = JobState.Aborted;
            break;
          case TaskState.Failed:
            state = JobState.Failed;
            break;
        }
      }
      if (!state.HasValue) {
        if (job.CalculatingCount > 0)
          state = JobState.Calculating;
        else if (waitingJobs > 0)
          state = JobState.Waiting;
        else
          state = JobState.Finished;
      }
      return new Model.Job() { Id = job.Id.ToString(), Name = job.Name, Resource = job.ResourceNames, State = state.Value };
    }


    public Model.Job GetJob(User user, string id) {
      HiveServiceLocator.Instance.Username = user.Username;
      HiveServiceLocator.Instance.Password = user.Password;
      HiveServiceLocator.Instance.EndpointConfigurationName = "WSHttpBinding_IHiveService";
      var guid = Guid.Parse(id);
      return ConvertJob(user, HiveServiceLocator.Instance.CallHiveService<HeuristicLab.Clients.Hive.Job>(s => s.GetJob(guid)));
    }


    public void DeleteJob(User user, string id) {
      HiveServiceLocator.Instance.Username = user.Username;
      HiveServiceLocator.Instance.Password = user.Password;
      HiveServiceLocator.Instance.EndpointConfigurationName = "WSHttpBinding_IHiveService";
      var guid = Guid.Parse(id);
      HiveServiceLocator.Instance.CallHiveService(s => s.DeleteJob(guid));      
    }

    public IList<Model.Run> GetJobResults(User user, string id) {
      HiveServiceLocator.Instance.Username = user.Username;
      HiveServiceLocator.Instance.Password = user.Password;
      HiveServiceLocator.Instance.EndpointConfigurationName = "WSHttpBinding_IHiveService";
      var guid = Guid.Parse(id);
      var jobTasks = HiveServiceLocator.Instance.CallHiveService<IEnumerable<HeuristicLab.Clients.Hive.LightweightTask>>(s => s.GetLightweightJobTasks(guid));

      IList<Guid> taskIds = new List<Guid>();
      foreach (var task in jobTasks) {
        taskIds.Add(task.Id);
      }
      TaskDownloader downloader = new TaskDownloader(taskIds);
      downloader.StartAsync();
      while (!downloader.IsFinished) {        
        Thread.Sleep(500);

        if (downloader.IsFaulted) {
          throw downloader.Exception;
        }
      }
      IDictionary<Guid, HiveTask> hiveTasks = downloader.Results;
      IList<Model.Run> runs = new List<Model.Run>();
      foreach (var keyTask in hiveTasks.Keys) {
        var oht = hiveTasks[keyTask] as OptimizerHiveTask;
        if (oht != null) {
          foreach (var run in oht.ItemTask.Item.Runs) {
            Model.Run taskRun = new Model.Run();
            taskRun.Id = taskRun.Name = run.Name;
            IList<Parameter> resultValues = new List<Model.Parameter>();
            foreach (var key in run.Results.Keys) {
              var value = run.Results[key];              
              Parameter result = MapHiveDataType(key, value);
              resultValues.Add(result);
            }
            taskRun.Results = resultValues;
            runs.Add(taskRun);
          }
        }
      }
      return runs;
    }

    //TODO: We might need images / +++
    private Parameter MapHiveDataType(string name, IItem item) {
      Parameter result = new Parameter();
      result.Type = ParameterType.String;
      if (item is IStringConvertibleValue) {
        var value = (item as IStringConvertibleValue).GetValue();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value };
      }
      else if (item is IStringConvertibleValueTuple) {
        var value1 = (item as IStringConvertibleValueTuple).Item1.GetValue();
        var value2 = (item as IStringConvertibleValueTuple).Item2.GetValue();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = "{" + value1 + ", " + value2 + "}" };
      }
      else if (item is IStringConvertibleArray) {
        StringBuilder sb = new StringBuilder();
        var array = item as IStringConvertibleArray;
        if (array.Length == 0) {
          sb.Append("[ ]");
        } else {
          sb.Append("[");
          for (int i=0; i < array.Length - 1; i++) {
            sb.Append(array.GetValue(i)).Append(", ");          
          }
          sb.Append(array.GetValue(array.Length - 1));
          sb.Append(" ]");
        }
        var value = sb.ToString();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value };
      }
      else if (item is IStringConvertibleMatrix) {
        StringBuilder sb = new StringBuilder();
        var matrix = item as IStringConvertibleMatrix;
        if (matrix.Rows == 0 || matrix.Columns == 0) {
          sb.Append("[ ]");
        }
        else {
          sb.Append("[ ");
          for (int r = 0; r < matrix.Rows; r++) {
            sb.Append("( ");
            for (int c = 0; c < matrix.Columns - 1; c++) {
              sb.Append(matrix.GetValue(r, c)).Append(", ");
            }
            sb.Append(matrix.GetValue(r, matrix.Columns - 1)).Append(r < matrix.Rows - 1 ? " ), " : " )");
          }
          sb.Append(" ]");
        }
        var value = sb.ToString();
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value };
      }
      else {
        result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = "Cannot be displayed as string" };
      }
      return result;
    }
  }
}