#region License Information /* HeuristicLab * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL) * * This file is part of HeuristicLab. * * HeuristicLab is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * HeuristicLab is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with HeuristicLab. If not, see . */ #endregion using System; using System.IO; using System.Threading; using HeuristicLab.Clients.Hive.SlaveCore.Properties; using HeuristicLab.Common; using HeuristicLab.Core; using HeuristicLab.PluginInfrastructure.Sandboxing; namespace HeuristicLab.Clients.Hive.SlaveCore { /// /// Manages a single job and it's appdomain. /// public class SlaveJob : MarshalByRefObject { private Executor executor; private AppDomain appDomain; private Semaphore waitForStartBeforeKillSem; private bool executorMonitoringRun; private Thread executorMonitoringThread; private PluginManager pluginManager; private ILog log; public Guid JobId { get; private set; } public bool IsPrepared { get; private set; } private int coresNeeded; public int CoresNeeded { get { return coresNeeded; } set { this.coresNeeded = value; } } public TimeSpan ExecutionTime { get { return executor != null ? executor.ExecutionTime : TimeSpan.Zero; } } public SlaveJob(PluginManager pluginManager, int coresNeeded, ILog log) { this.pluginManager = pluginManager; this.coresNeeded = coresNeeded; this.log = log; waitForStartBeforeKillSem = new Semaphore(0, 1); executorMonitoringRun = true; IsPrepared = false; } public void StartJobAsync(Job job, JobData jobData) { try { this.JobId = job.Id; Prepare(job); StartJobInAppDomain(jobData); } catch (Exception) { // make sure to clean up if something went wrong DisposeAppDomain(); throw; } } public void PauseJob() { if (!IsPrepared) throw new AppDomainNotCreatedException(); if (!executor.IsPausing && !executor.IsStopping) executor.Pause(); } public void StopJob() { if (!IsPrepared) throw new AppDomainNotCreatedException(); if (!executor.IsPausing && !executor.IsStopping) executor.Stop(); } private void Prepare(Job job) { string pluginDir = Path.Combine(pluginManager.PluginTempBaseDir, job.Id.ToString()); string configFileName; pluginManager.PreparePlugins(job, out configFileName); appDomain = CreateAppDomain(job, pluginDir, configFileName); IsPrepared = true; } private AppDomain CreateAppDomain(Job job, String pluginDir, string configFileName) { if (job.IsPrivileged) { appDomain = SandboxManager.CreateAndInitPrivilegedSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName)); } else { appDomain = SandboxManager.CreateAndInitSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName)); } appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException); log.LogMessage("Creating AppDomain"); executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName); executor.JobId = job.Id; executor.CoresNeeded = job.CoresNeeded; executor.MemoryNeeded = job.MemoryNeeded; return appDomain; } private void StartJobInAppDomain(JobData jobData) { executor.Start(jobData.Data); waitForStartBeforeKillSem.Release(); StartExecutorMonitoringThread(); } public void DisposeAppDomain() { log.LogMessage(string.Format("Shutting down Appdomain for Job {0}", JobId)); StopExecutorMonitoringThread(); if (executor != null) { executor.Dispose(); } if (appDomain != null) { appDomain.UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException); int repeat = Settings.Default.PluginDeletionRetries; while (repeat > 0) { try { waitForStartBeforeKillSem.WaitOne(); AppDomain.Unload(appDomain); waitForStartBeforeKillSem.Dispose(); repeat = 0; } catch (CannotUnloadAppDomainException) { log.LogMessage("Could not unload AppDomain, will try again in 1 sec."); Thread.Sleep(Settings.Default.PluginDeletionTimeout); repeat--; if (repeat == 0) { throw; // rethrow and let app crash } } } } pluginManager.DeletePluginsForJob(JobId); GC.Collect(); } private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) { DisposeAppDomain(); OnExceptionOccured(new Exception("Unhandled exception: " + e.ExceptionObject.ToString())); } public JobData GetJobData() { return executor.GetJobData(); } #region ExecutorMonitorThread private void StartExecutorMonitoringThread() { executorMonitoringThread = new Thread(MonitorExecutor); executorMonitoringThread.Start(); } private void StopExecutorMonitoringThread() { if (executorMonitoringThread != null) { if (executorMonitoringRun) { executorMonitoringRun = false; executor.ExecutorCommandQueue.AddMessage(ExecutorMessageType.StopExecutorMonitoringThread); } } } /// /// Because the executor is in an appdomain and is not able to call back /// (because of security -> lease time for marshall-by-ref object is 5 min), /// we have to poll the executor for events we have to react to (e.g. job finished...) /// private void MonitorExecutor() { while (executorMonitoringRun) { //this blocks through the appdomain border, that's why the lease gets renewed ExecutorMessage message = executor.ExecutorCommandQueue.GetMessage(); switch (message.MessageType) { case ExecutorMessageType.JobStarted: OnJobStarted(); break; case ExecutorMessageType.JobPaused: executorMonitoringRun = false; OnJobPaused(); DisposeAppDomain(); break; case ExecutorMessageType.JobStopped: executorMonitoringRun = false; OnJobStopped(); DisposeAppDomain(); break; case ExecutorMessageType.JobFailed: executorMonitoringRun = false; OnJobFailed(new JobFailedException(executor.CurrentException)); DisposeAppDomain(); break; case ExecutorMessageType.StopExecutorMonitoringThread: executorMonitoringRun = false; return; } } } #endregion public event EventHandler> JobStarted; private void OnJobStarted() { var handler = JobStarted; if (handler != null) handler(this, new EventArgs(this.JobId)); } public event EventHandler> JobStopped; private void OnJobStopped() { var handler = JobStopped; if (handler != null) handler(this, new EventArgs(this.JobId)); } public event EventHandler> JobPaused; private void OnJobPaused() { var handler = JobPaused; if (handler != null) handler(this, new EventArgs(this.JobId)); } public event EventHandler> JobAborted; private void OnJobAborted() { var handler = JobAborted; if (handler != null) handler(this, new EventArgs(this.JobId)); } public event EventHandler> JobFailed; private void OnJobFailed(Exception exception) { var handler = JobFailed; if (handler != null) handler(this, new EventArgs(this.JobId, exception)); } public event EventHandler> ExceptionOccured; private void OnExceptionOccured(Exception exception) { var handler = ExceptionOccured; if (handler != null) handler(this, new EventArgs(this.JobId, exception)); } } }