#region License Information
/* HeuristicLab
* Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
*
* This file is part of HeuristicLab.
*
* HeuristicLab is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* HeuristicLab is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HeuristicLab. If not, see .
*/
#endregion
using System;
using System.IO;
using System.Threading;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.PluginInfrastructure.Sandboxing;
namespace HeuristicLab.Clients.Hive.SlaveCore {
///
/// Manages a single job and it's appdomain.
///
public class SlaveJob : MarshalByRefObject {
private Executor executor;
private AppDomain appDomain;
private Semaphore waitForStartBeforeKillSem;
private bool executorMonitoringRun;
private Thread executorMonitoringThread;
private PluginManager pluginManager;
private ILog log;
public Guid JobId { get; private set; }
public bool IsPrepared { get; private set; }
private int coresNeeded;
public int CoresNeeded {
get { return coresNeeded; }
set { this.coresNeeded = value; }
}
public TimeSpan ExecutionTime {
get { return executor != null ? executor.ExecutionTime : TimeSpan.Zero; }
}
public SlaveJob(PluginManager pluginManager, int coresNeeded, ILog log) {
this.pluginManager = pluginManager;
this.coresNeeded = coresNeeded;
this.log = log;
waitForStartBeforeKillSem = new Semaphore(0, 1);
executorMonitoringRun = true;
IsPrepared = false;
}
public void StartJobAsync(Job job, JobData jobData) {
try {
this.JobId = job.Id;
Prepare(job);
StartJobInAppDomain(jobData);
}
catch (Exception) {
// make sure to clean up if something went wrong
DisposeAppDomain();
throw;
}
}
public void PauseJob() {
if (!IsPrepared) throw new AppDomainNotCreatedException();
if (!executor.IsPausing && !executor.IsStopping) executor.Pause();
}
public void StopJob() {
if (!IsPrepared) throw new AppDomainNotCreatedException();
if (!executor.IsPausing && !executor.IsStopping) executor.Stop();
}
private void Prepare(Job job) {
string pluginDir = Path.Combine(pluginManager.PluginTempBaseDir, job.Id.ToString());
string configFileName;
pluginManager.PreparePlugins(job, out configFileName);
appDomain = CreateAppDomain(job, pluginDir, configFileName);
IsPrepared = true;
}
private AppDomain CreateAppDomain(Job job, String pluginDir, string configFileName) {
if (job.IsPrivileged) {
appDomain = SandboxManager.CreateAndInitPrivilegedSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
} else {
appDomain = SandboxManager.CreateAndInitSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
}
appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
log.LogMessage("Creating AppDomain");
executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
executor.JobId = job.Id;
executor.CoresNeeded = job.CoresNeeded;
executor.MemoryNeeded = job.MemoryNeeded;
return appDomain;
}
private void StartJobInAppDomain(JobData jobData) {
executor.Start(jobData.Data);
waitForStartBeforeKillSem.Release();
StartExecutorMonitoringThread();
}
public void DisposeAppDomain() {
log.LogMessage(string.Format("Shutting down Appdomain for Job {0}", JobId));
StopExecutorMonitoringThread();
if (executor != null) {
executor.Dispose();
}
if (appDomain != null) {
appDomain.UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
int repeat = 5;
while (repeat > 0) {
try {
waitForStartBeforeKillSem.WaitOne();
AppDomain.Unload(appDomain);
waitForStartBeforeKillSem.Dispose();
repeat = 0;
}
catch (CannotUnloadAppDomainException) {
log.LogMessage("Could not unload AppDomain, will try again in 1 sec.");
Thread.Sleep(1000);
repeat--;
if (repeat == 0) {
throw; // rethrow and let app crash
}
}
}
}
pluginManager.DeletePluginsForJob(JobId);
GC.Collect();
}
private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
DisposeAppDomain();
OnExceptionOccured(new Exception("Unhandled exception: " + e.ExceptionObject.ToString()));
}
public JobData GetJobData() {
return executor.GetJobData();
}
#region ExecutorMonitorThread
private void StartExecutorMonitoringThread() {
executorMonitoringThread = new Thread(MonitorExecutor);
executorMonitoringThread.Start();
}
private void StopExecutorMonitoringThread() {
if (executorMonitoringThread != null) {
if (executorMonitoringRun) {
executorMonitoringRun = false;
executor.ExecutorCommandQueue.AddMessage(ExecutorMessageType.StopExecutorMonitoringThread);
}
}
}
///
/// Because the executor is in an appdomain and is not able to call back
/// (because of security -> lease time for marshall-by-ref object is 5 min),
/// we have to poll the executor for events we have to react to (e.g. job finished...)
///
private void MonitorExecutor() {
while (executorMonitoringRun) {
//this blocks through the appdomain border, that's why the lease gets renewed
ExecutorMessage message = executor.ExecutorCommandQueue.GetMessage();
switch (message.MessageType) {
case ExecutorMessageType.JobStarted:
OnJobStarted();
break;
case ExecutorMessageType.JobPaused:
executorMonitoringRun = false;
OnJobPaused();
DisposeAppDomain();
break;
case ExecutorMessageType.JobStopped:
executorMonitoringRun = false;
OnJobStopped();
DisposeAppDomain();
break;
case ExecutorMessageType.JobFailed:
executorMonitoringRun = false;
OnJobFailed(new JobFailedException(executor.CurrentException));
DisposeAppDomain();
break;
case ExecutorMessageType.StopExecutorMonitoringThread:
executorMonitoringRun = false;
return;
}
}
}
#endregion
public event EventHandler> JobStarted;
private void OnJobStarted() {
var handler = JobStarted;
if (handler != null) handler(this, new EventArgs(this.JobId));
}
public event EventHandler> JobStopped;
private void OnJobStopped() {
var handler = JobStopped;
if (handler != null) handler(this, new EventArgs(this.JobId));
}
public event EventHandler> JobPaused;
private void OnJobPaused() {
var handler = JobPaused;
if (handler != null) handler(this, new EventArgs(this.JobId));
}
public event EventHandler> JobAborted;
private void OnJobAborted() {
var handler = JobAborted;
if (handler != null) handler(this, new EventArgs(this.JobId));
}
public event EventHandler> JobFailed;
private void OnJobFailed(Exception exception) {
var handler = JobFailed;
if (handler != null) handler(this, new EventArgs(this.JobId, exception));
}
public event EventHandler> ExceptionOccured;
private void OnExceptionOccured(Exception exception) {
var handler = ExceptionOccured;
if (handler != null) handler(this, new EventArgs(this.JobId, exception));
}
}
}