source: branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveJob.cs @ 6360

using System;
using System.IO;
using System.Threading;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Hive;
using HeuristicLab.PluginInfrastructure.Sandboxing;

namespace HeuristicLab.Clients.Hive.SlaveCore {

  public class SlaveJob : MarshalByRefObject {
    private Executor executor;
    private AppDomain appDomain;
    private Semaphore waitForStartBeforeKillSem;
    private bool executorMonitoringRun;
    private Thread executorMonitoringThread;
    private PluginManager pluginManager;
    private ILog log;

    public Guid JobId { get; private set; }
    public bool IsPrepared { get; private set; }

    private int coresNeeded;
    public int CoresNeeded {
      get { return coresNeeded; }
      set { this.coresNeeded = value; }
    }

    public TimeSpan ExecutionTime {
      get { return executor != null ? executor.ExecutionTime : TimeSpan.Zero; }
    }

    public SlaveJob(PluginManager pluginManager, int coresNeeded, ILog log) {
      this.pluginManager = pluginManager;
      this.coresNeeded = coresNeeded;
      this.log = log;
      waitForStartBeforeKillSem = new Semaphore(0, 1);
      executorMonitoringRun = true;
      IsPrepared = false;
    }

    public void StartJobAsync(Job job, JobData jobData) {
      try {
        this.JobId = job.Id;
        Prepare(job);
        StartJobInAppDomain(jobData);
      }
      catch (Exception) {
        DisposeAppDomain(); // make sure to clean up if something went wrong
        throw;
      }
    }

    public void PauseJob() {
      if (!IsPrepared) throw new AppDomainNotCreatedException();
      if (!executor.IsPausing && !executor.IsStopping) executor.Pause();
    }

    public void StopJob() {
      if (!IsPrepared) throw new AppDomainNotCreatedException();
      if (!executor.IsPausing && !executor.IsStopping) executor.Stop();
    }

    private void Prepare(Job job) {
      string pluginDir = Path.Combine(pluginManager.PluginTempBaseDir, job.Id.ToString());
      string configFileName;
      pluginManager.PreparePlugins(job, out configFileName);
      appDomain = CreateAppDomain(job, pluginDir, configFileName);
      IsPrepared = true;
    }

    private AppDomain CreateAppDomain(Job job, String pluginDir, string configFileName) {
      appDomain = SandboxManager.CreateAndInitPrivilegedSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
      appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);

      log.LogMessage("Creating AppDomain");
      executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);

      executor.JobId = job.Id;
      executor.CoresNeeded = job.CoresNeeded;
      executor.MemoryNeeded = job.MemoryNeeded;
      return appDomain;
    }

    private void StartJobInAppDomain(JobData jobData) {
      executor.Start(jobData.Data);
      waitForStartBeforeKillSem.Release();
      StartExecutorMonitoringThread();
    }

    /// <summary>
    /// Kill a appdomain with a specific id.
    /// </summary>
    /// <param name="JobId">the GUID of the job</param>    
    public void DisposeAppDomain() {
      log.LogMessage(string.Format("Shutting down Appdomain for Job {0}", JobId));
      StopExecutorMonitoringThread();

      if (executor != null) {
        executor.Dispose();
      }

      if (appDomain != null) {
        appDomain.UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
        int repeat = 5;
        while (repeat > 0) {
          try {
            waitForStartBeforeKillSem.WaitOne();
            AppDomain.Unload(appDomain);
            waitForStartBeforeKillSem.Dispose();
            repeat = 0;
          }
          catch (CannotUnloadAppDomainException) {
            log.LogMessage("Could not unload AppDomain, will try again in 1 sec.");
            Thread.Sleep(1000);
            repeat--;
            if (repeat == 0) {
              throw; // rethrow and let app crash
            }
          }
        }
      }
      pluginManager.DeletePluginsForJob(JobId);
      GC.Collect();
    }

    private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
      DisposeAppDomain();
      OnExceptionOccured(new Exception("Unhandled exception: " +e.ExceptionObject.ToString()));
    }

    public JobData GetJobData() {
      return executor.GetJobData();
    }

    #region ExecutorMonitorThread
    private void StartExecutorMonitoringThread() {
      executorMonitoringThread = new Thread(MonitorExecutor);
      executorMonitoringThread.Start();
    }

    private void StopExecutorMonitoringThread() {
      if (executorMonitoringThread != null) {
        if (executorMonitoringRun) {
          executorMonitoringRun = false;
          executor.executorQueue.AddMessage(ExecutorMessageType.StopExecutorMonitoringThread);
        }
      }
    }

    /// <summary>
    /// Because the executor is in an appdomain and is not able to call back 
    /// (because of security -> lease time for marshall-by-ref object is 5 min),
    /// we have to poll the executor for events we have to react to (e.g. job finished...)    
    /// </summary>
    private void MonitorExecutor() {
      while (executorMonitoringRun) {
        //this blocks through the appdomain border, that's why the lease gets renewed
        ExecutorMessage message = executor.executorQueue.GetMessage();

        switch (message.MessageType) {
          case ExecutorMessageType.JobStarted:
            OnJobStarted();
            break;

          case ExecutorMessageType.JobPaused:
            executorMonitoringRun = false;
            OnJobPaused();
            DisposeAppDomain();
            break;

          case ExecutorMessageType.JobStopped:
            executorMonitoringRun = false;
            OnJobStopped();
            DisposeAppDomain();
            break;

          case ExecutorMessageType.JobFailed:
            executorMonitoringRun = false;
            OnJobFailed(new JobFailedException(executor.CurrentException));
            DisposeAppDomain();
            break;

          case ExecutorMessageType.NewChildJob:
            OnNewChildJob(((JobExecutorMessage)message).Job);
            break;

          case ExecutorMessageType.WaitForChildJobs:
            executorMonitoringRun = false;
            OnWaitForChildJobs();
            DisposeAppDomain();
            break;

          case ExecutorMessageType.DeleteChildJobs:
            OnDeleteChildJobs();
            break;

          case ExecutorMessageType.StopExecutorMonitoringThread:
            executorMonitoringRun = false;
            return;
        }
      }
    }
    #endregion

    public event EventHandler<EventArgs<Guid>> JobStarted;
    private void OnJobStarted() {
      var handler = JobStarted;
      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
    }

    public event EventHandler<EventArgs<Guid>> JobStopped;
    private void OnJobStopped() {
      var handler = JobStopped;
      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
    }

    public event EventHandler<EventArgs<Guid>> JobPaused;
    private void OnJobPaused() {
      var handler = JobPaused;
      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
    }

    public event EventHandler<EventArgs<Guid>> JobAborted;
    private void OnJobAborted() {
      var handler = JobAborted;
      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
    }

    public event EventHandler<EventArgs<Guid, Exception>> JobFailed;
    private void OnJobFailed(Exception exception) {
      var handler = JobFailed;
      if (handler != null) handler(this, new EventArgs<Guid, Exception>(this.JobId, exception));
    }

    public event EventHandler<EventArgs<Guid, Exception>> ExceptionOccured;
    private void OnExceptionOccured(Exception exception) {
      var handler = ExceptionOccured;
      if (handler != null) handler(this, new EventArgs<Guid, Exception>(this.JobId, exception));
    }

    public event EventHandler<EventArgs<Guid, IJob>> NewChildJob;
    private void OnNewChildJob(IJob job) {
      var handler = NewChildJob;
      if (handler != null) handler(this, new EventArgs<Guid, IJob>(this.JobId, job));
    }

    public event EventHandler<EventArgs<Guid>> WaitForChildJobs;
    private void OnWaitForChildJobs() {
      var handler = WaitForChildJobs;
      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
    }

    public event EventHandler<EventArgs<Guid>> DeleteChildJobs;
    private void OnDeleteChildJobs() {
      var handler = DeleteChildJobs;
      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
    }
  }
}