#region License Information
/* HeuristicLab
* Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
*
* This file is part of HeuristicLab.
*
* HeuristicLab is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* HeuristicLab is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HeuristicLab. If not, see .
*/
#endregion
using System;
using System.IO;
using System.Threading;
using HeuristicLab.Clients.Hive.SlaveCore.Properties;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.PluginInfrastructure.Sandboxing;
namespace HeuristicLab.Clients.Hive.SlaveCore {
///
/// Manages a single task and it's appdomain.
///
public class SlaveTask : MarshalByRefObject {
private Executor executor;
private AppDomain appDomain;
private Semaphore waitForStartBeforeKillSem;
private bool executorMonitoringRun;
private Thread executorMonitoringThread;
private PluginManager pluginManager;
private ILog log;
public Guid TaskId { get; private set; }
public bool IsPrepared { get; private set; }
private TaskData originalTaskData;
private int coresNeeded;
public int CoresNeeded {
get { return coresNeeded; }
set { this.coresNeeded = value; }
}
public TimeSpan ExecutionTime {
get {
try {
return executor != null ? executor.ExecutionTime : TimeSpan.Zero;
}
catch (Exception ex) {
EventLogManager.LogException(ex);
return TimeSpan.Zero;
}
}
}
public SlaveTask(PluginManager pluginManager, int coresNeeded, ILog log) {
this.pluginManager = pluginManager;
this.coresNeeded = coresNeeded;
this.log = log;
waitForStartBeforeKillSem = new Semaphore(0, 1);
executorMonitoringRun = true;
IsPrepared = false;
}
public void StartJobAsync(Task task, TaskData taskData) {
try {
this.TaskId = task.Id;
originalTaskData = taskData;
Prepare(task);
StartTaskInAppDomain(taskData);
}
catch (Exception) {
// make sure to clean up if something went wrong
DisposeAppDomain();
throw;
}
}
public void PauseTask() {
if (!IsPrepared) throw new AppDomainNotCreatedException();
if (!executor.IsPausing && !executor.IsStopping) executor.Pause();
}
public void StopTask() {
if (!IsPrepared) throw new AppDomainNotCreatedException();
if (!executor.IsPausing && !executor.IsStopping) executor.Stop();
}
private void Prepare(Task task) {
string pluginDir = Path.Combine(pluginManager.PluginTempBaseDir, task.Id.ToString());
string configFileName;
pluginManager.PreparePlugins(task, out configFileName);
appDomain = CreateAppDomain(task, pluginDir, configFileName);
IsPrepared = true;
}
private AppDomain CreateAppDomain(Task task, String pluginDir, string configFileName) {
appDomain = SandboxManager.CreateAndInitSandbox(task.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
log.LogMessage("Creating AppDomain");
executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
executor.TaskId = task.Id;
executor.CoresNeeded = task.CoresNeeded;
executor.MemoryNeeded = task.MemoryNeeded;
return appDomain;
}
private void StartTaskInAppDomain(TaskData taskData) {
executor.Start(taskData.Data);
waitForStartBeforeKillSem.Release();
StartExecutorMonitoringThread();
}
public void DisposeAppDomain() {
log.LogMessage(string.Format("Shutting down Appdomain for Task {0}", TaskId));
StopExecutorMonitoringThread();
if (executor != null) {
try {
executor.Dispose();
}
catch (Exception ex) {
EventLogManager.LogException(ex);
}
}
if (appDomain != null) {
appDomain.UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
int repeat = Settings.Default.PluginDeletionRetries;
while (repeat > 0) {
try {
waitForStartBeforeKillSem.WaitOne(Settings.Default.ExecutorSemTimeouts);
AppDomain.Unload(appDomain);
waitForStartBeforeKillSem.Dispose();
repeat = 0;
}
catch (CannotUnloadAppDomainException) {
log.LogMessage("Could not unload AppDomain, will try again in 1 sec.");
Thread.Sleep(Settings.Default.PluginDeletionTimeout);
repeat--;
if (repeat == 0) {
throw; // rethrow and let app crash
}
}
}
}
pluginManager.DeletePluginsForJob(TaskId);
GC.Collect();
}
private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
DisposeAppDomain();
OnTaskFailed(new Exception("Unhandled exception: " + e.ExceptionObject.ToString()));
}
public TaskData GetTaskData() {
TaskData data = null;
try {
data = executor.GetTaskData();
//this means that there was a problem executing the task
if (data == null) return originalTaskData;
}
catch (Exception ex) {
EventLogManager.LogException(ex);
}
return data;
}
#region ExecutorMonitorThread
private void StartExecutorMonitoringThread() {
executorMonitoringThread = new Thread(MonitorExecutor);
executorMonitoringThread.Start();
}
private void StopExecutorMonitoringThread() {
if (executorMonitoringThread != null) {
if (executorMonitoringRun) {
executorMonitoringRun = false;
executor.ExecutorCommandQueue.AddMessage(ExecutorMessageType.StopExecutorMonitoringThread);
}
}
}
///
/// Because the executor is in an appdomain and is not able to call back
/// (because of security -> lease time for marshall-by-ref object is 5 min),
/// we have to poll the executor for events we have to react to (e.g. task finished...)
///
private void MonitorExecutor() {
while (executorMonitoringRun) {
//this call goes through the appdomain border. We have to
//poll so that the the lease gets renewed
ExecutorMessage message;
do {
message = executor.ExecutorCommandQueue.GetMessage();
} while (message == null);
switch (message.MessageType) {
case ExecutorMessageType.TaskStarted:
OnTaskStarted();
break;
case ExecutorMessageType.TaskPaused:
executorMonitoringRun = false;
OnTaskPaused();
DisposeAppDomain();
break;
case ExecutorMessageType.TaskStopped:
executorMonitoringRun = false;
OnTaskStopped();
DisposeAppDomain();
break;
case ExecutorMessageType.TaskFailed:
executorMonitoringRun = false;
OnTaskFailed(new TaskFailedException(executor.CurrentExceptionStr));
DisposeAppDomain();
break;
case ExecutorMessageType.StopExecutorMonitoringThread:
executorMonitoringRun = false;
break;
}
}
}
#endregion
public event EventHandler> TaskStarted;
private void OnTaskStarted() {
var handler = TaskStarted;
if (handler != null) handler(this, new EventArgs(this.TaskId));
}
public event EventHandler> TaskStopped;
private void OnTaskStopped() {
var handler = TaskStopped;
if (handler != null) handler(this, new EventArgs(this.TaskId));
}
public event EventHandler> TaskPaused;
private void OnTaskPaused() {
var handler = TaskPaused;
if (handler != null) handler(this, new EventArgs(this.TaskId));
}
public event EventHandler> TaskAborted;
private void OnTaskAborted() {
var handler = TaskAborted;
if (handler != null) handler(this, new EventArgs(this.TaskId));
}
public event EventHandler> TaskFailed;
private void OnTaskFailed(Exception exception) {
var handler = TaskFailed;
if (handler != null) handler(this, new EventArgs(this.TaskId, exception));
}
}
}