Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Views/3.4/SlaveView.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Views/3.4/SlaveView.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Views/3.4/SlaveView.cs	(revision 6203)
@@ -124,5 +124,5 @@
       DataPoint pJobs = new DataPoint(status.Jobs.Count, status.Jobs.Count);
       DataPoint pJobsAborted = new DataPoint(status.JobsAborted, status.JobsAborted);
-      DataPoint pJobsDone = new DataPoint(status.JobsDone, status.JobsDone);
+      DataPoint pJobsDone = new DataPoint(status.JobsFinished, status.JobsFinished);
       DataPoint pJobsFetched = new DataPoint(status.JobsFetched, status.JobsFetched);
 
@@ -131,5 +131,5 @@
       pJobsAborted.LegendText = "Aborted jobs: " + status.JobsAborted;
       pJobsAborted.Color = System.Drawing.Color.Red;
-      pJobsDone.LegendText = "Finished jobs: " + status.JobsDone;
+      pJobsDone.LegendText = "Finished jobs: " + status.JobsFinished;
       pJobsDone.Color = System.Drawing.Color.Green;
       pJobsFetched.LegendText = "Fetched jobs: " + status.JobsFetched;
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ConfigManager.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ConfigManager.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ConfigManager.cs	(revision 6203)
@@ -79,17 +79,20 @@
 
       st.TotalCores = slave.Cores.HasValue ? slave.Cores.Value : 0;
-      st.FreeCores = slave.Cores.HasValue ? slave.Cores.Value - GetUsedCores() : 0;
+      st.FreeCores = slave.Cores.HasValue ? slave.Cores.Value - SlaveStatusInfo.UsedCores : 0;
 
       st.JobsAborted = SlaveStatusInfo.JobsAborted;
-      st.JobsDone = SlaveStatusInfo.JobsProcessed;
+      st.JobsFinished = SlaveStatusInfo.JobsFinished;
       st.JobsFetched = SlaveStatusInfo.JobsFetched;
-
-      Dictionary<Guid, Executor> engines = Core.Executors;
+      st.JobsFailed = SlaveStatusInfo.JobsFailed;
+
+      Dictionary<Guid, SlaveJob> slaveJobs = Core.SlaveJobs;
       st.Jobs = new List<JobStatus>();
 
-      lock (engines) {
-        foreach (KeyValuePair<Guid, Executor> kvp in engines) {
-          Executor e = kvp.Value;
-          st.Jobs.Add(new JobStatus { JobId = e.JobId, ExecutionTime = e.ExecutionTime, Since = e.CreationTime });
+      lock (slaveJobs) {
+        foreach (KeyValuePair<Guid, SlaveJob> kvp in slaveJobs) {
+          Executor e = kvp.Value.JobExecutor;
+          if (e != null && !kvp.Value.Finished) {
+            st.Jobs.Add(new JobStatus { JobId = e.JobId, ExecutionTime = e.ExecutionTime, Since = e.CreationTime });
+          }
         }
       }
@@ -99,19 +102,17 @@
     public Dictionary<Guid, TimeSpan> GetExecutionTimeOfAllJobs() {
       Dictionary<Guid, TimeSpan> prog = new Dictionary<Guid, TimeSpan>();
-      Dictionary<Guid, Executor> engines = Core.Executors;
-      lock (engines) {
-        foreach (KeyValuePair<Guid, Executor> kvp in engines) {
-          Executor e = kvp.Value;
-          //don't include jobs in hb's which are currently serializing
-          if (e.SendHeartbeatForExecutor) {
-            prog[e.JobId] = e.ExecutionTime;
+      Dictionary<Guid, SlaveJob> slaveJobs = Core.SlaveJobs;
+      lock (slaveJobs) {
+        foreach (KeyValuePair<Guid, SlaveJob> kvp in slaveJobs) {
+          Executor e = kvp.Value.JobExecutor;
+          if (e != null && !kvp.Value.Finished) {
+            //don't include jobs in hb's which are currently serializing
+            if (e.SendHeartbeatForExecutor) {
+              prog[e.JobId] = e.ExecutionTime;
+            }
           }
         }
       }
       return prog;
-    }
-
-    public int GetUsedCores() {
-      return Core.GetCoresNeeded();
     }
 
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs	(revision 6203)
@@ -23,6 +23,4 @@
 using System.Collections.Generic;
 using System.Diagnostics;
-using System.IO;
-using System.Linq;
 using System.ServiceModel;
 using System.Threading;
@@ -45,19 +43,15 @@
     public static ILog Log { get; set; }
 
-    private Dictionary<Guid, Executor> executors = new Dictionary<Guid, Executor>();
-    private Dictionary<Guid, AppDomain> appDomains = new Dictionary<Guid, AppDomain>();
-
-    // signalizes if the Executor.Start method has properly finished. only then the appdomain may be unloaded
-    private Dictionary<Guid, Semaphore> semaphores = new Dictionary<Guid, Semaphore>();
+    private Dictionary<Guid, SlaveJob> slaveJobs = new Dictionary<Guid, SlaveJob>();
 
     private WcfService wcfService;
-    private HeartbeatManager heartbeatManager;
-    private int coreThreadId;
+    private static HeartbeatManager heartbeatManager;
+    public static HeartbeatManager HBManager { get { return heartbeatManager; } }
 
     private ISlaveCommunication clientCom;
     private ServiceHost slaveComm;
 
-    public Dictionary<Guid, Executor> Executors {
-      get { return executors; }
+    public Dictionary<Guid, SlaveJob> SlaveJobs {
+      get { return slaveJobs; }
     }
 
@@ -68,5 +62,4 @@
     /// </summary>
     public void Start() {
-      coreThreadId = Thread.CurrentThread.ManagedThreadId;
       abortRequested = false;
 
@@ -160,12 +153,19 @@
             Task.Factory.StartNew((jobIdObj) => {
               Guid jobId = (Guid)jobIdObj;
-              Job job = wcfService.GetJob(jobId);
-              if (job == null) throw new JobNotFoundException(jobId);
-              JobData jobData = wcfService.GetJobData(job.Id);
-              if (jobData == null) throw new JobDataNotFoundException(jobId);
-              SlaveStatusInfo.JobsFetched++;
-              job = wcfService.UpdateJobState(job.Id, JobState.Calculating, null);
-              if (job == null) throw new JobNotFoundException(jobId);
-              StartJobInAppDomain(job, jobData);
+              SlaveJob newJob = new SlaveJob(this);
+              bool start = true;
+
+              lock (slaveJobs) {
+                if (slaveJobs.ContainsKey(jobId)) {
+                  start = false;
+                  clientCom.LogMessage(string.Format("Job with id {0} already exists. Start aborted.", jobId));
+                } else {
+                  slaveJobs.Add(jobId, newJob);
+                }
+              }
+
+              if (start) {
+                newJob.CalculateJob(jobId);
+              }
             }, container.JobId)
             .ContinueWith((t) => {
@@ -173,5 +173,5 @@
               clientCom.LogMessage(t.Exception.ToString());
               wcfService.UpdateJobState(container.JobId, JobState.Failed, t.Exception.ToString());
-              SlaveStatusInfo.JobsAborted++;
+              SlaveStatusInfo.IncrementJobsFailed();
             }, TaskContinuationOptions.OnlyOnFaulted);
             break;
@@ -189,12 +189,73 @@
             break;
           case MessageContainer.MessageType.AbortJob:
-            SlaveStatusInfo.JobsAborted++;
-            KillAppDomain(container.JobId);
+            SlaveStatusInfo.IncrementJobsAborted(); //TODO: move to a sane place
+
+            Task.Factory.StartNew((jobIdObj) => {
+              Guid jobId = (Guid)jobIdObj;
+              bool abort = true;
+              SlaveJob sj = null;
+
+              lock (slaveJobs) {
+                if (!slaveJobs.ContainsKey(jobId)) {
+                  clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Abort aborted.", jobId));
+                  abort = false;
+                } else {
+                  sj = slaveJobs[jobId];
+                }
+              }
+              if (abort && !sj.Finished) {
+                sj.KillAppDomain();
+              }
+            }, container.JobId)
+             .ContinueWith((t) => {
+               // handle exception of task
+               clientCom.LogMessage(t.Exception.ToString());
+             }, TaskContinuationOptions.OnlyOnFaulted);
             break;
           case MessageContainer.MessageType.StopJob:
-            DoStopJob(container.JobId);
+            Task.Factory.StartNew((jobIdObj) => {
+              Guid jobId = (Guid)jobIdObj;
+              bool stop = true;
+              SlaveJob sj = null;
+
+              lock (slaveJobs) {
+                if (!slaveJobs.ContainsKey(jobId)) {
+                  clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Stop aborted.", jobId));
+                  stop = false;
+                } else {
+                  sj = slaveJobs[jobId];
+                }
+              }
+              if (stop && !sj.Finished) {
+                sj.StopJob();
+              }
+            }, container.JobId)
+             .ContinueWith((t) => {
+               // handle exception of task
+               clientCom.LogMessage(t.Exception.ToString());
+             }, TaskContinuationOptions.OnlyOnFaulted);
             break;
           case MessageContainer.MessageType.PauseJob:
-            DoPauseJob(container.JobId);
+            Task.Factory.StartNew((jobIdObj) => {
+              Guid jobId = (Guid)jobIdObj;
+              bool pause = true;
+              SlaveJob sj = null;
+
+              lock (slaveJobs) {
+                if (!slaveJobs.ContainsKey(jobId)) {
+                  clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Pause aborted.", jobId));
+                  pause = false;
+                } else {
+                  sj = slaveJobs[jobId];
+                }
+              }
+              if (pause && !sj.Finished) {
+                sj.PauseJob();
+              }
+            }, container.JobId)
+             .ContinueWith((t) => {
+               // handle exception of task
+               clientCom.LogMessage(t.Exception.ToString());
+             }, TaskContinuationOptions.OnlyOnFaulted);
             break;
           case MessageContainer.MessageType.Restart:
@@ -213,75 +274,14 @@
     }
 
-    private void DoPauseJob(Guid jobId) {
-      if (!executors.ContainsKey(jobId)) {
-        clientCom.LogMessage("DoPauseJob: Can't find job with id " + jobId);
-      } else {
-        Job job = wcfService.GetJob(jobId);
-
-        if (job != null && executors.ContainsKey(job.Id)) {
-          executors[job.Id].Pause();
-          JobData sJob = executors[job.Id].GetPausedJob();
-          job.ExecutionTime = executors[job.Id].ExecutionTime;
-
-          try {
-            if (executors[job.Id].CurrentException != string.Empty) {
-              wcfService.UpdateJobState(job.Id, JobState.Failed, executors[job.Id].CurrentException);
-              SlaveStatusInfo.JobsAborted++;
-            } else {
-              SlaveStatusInfo.JobsProcessed++;
-            }
-            clientCom.LogMessage("Sending the paused job with id: " + job.Id);
-            wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Paused);
+    /// <summary>
+    /// aborts all running jobs, no results are sent back
+    /// </summary>
+    private void DoAbortAll() {
+      lock (slaveJobs) {
+        foreach (SlaveJob sj in slaveJobs.Values) {
+          if (!sj.Finished) {
+            sj.KillAppDomain();
           }
-          catch (Exception e) {
-            clientCom.LogMessage("Transmitting to server failed. Storing the paused job with id: " + job.Id + " to hdd (" + e.ToString() + ")");
-          }
-          finally {
-            KillAppDomain(job.Id); // kill app-domain in every case          
-          }
-        }
-      }
-    }
-
-    private void DoStopJob(Guid jobId) {
-      if (!executors.ContainsKey(jobId)) {
-        clientCom.LogMessage("DoStopJob: Can't find job with id " + jobId);
-      } else {
-        Job job = wcfService.GetJob(jobId);
-
-        if (job != null) {
-          executors[job.Id].Stop();
-          JobData sJob = executors[job.Id].GetFinishedJob();
-          job.ExecutionTime = executors[job.Id].ExecutionTime;
-
-          try {
-            if (executors[job.Id].CurrentException != string.Empty) {
-              wcfService.UpdateJobState(job.Id, JobState.Failed, executors[job.Id].CurrentException);
-            }
-            SlaveStatusInfo.JobsAborted++;
-
-            clientCom.LogMessage("Sending the stopped job with id: " + job.Id);
-            wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Aborted);
-          }
-          catch (Exception e) {
-            clientCom.LogMessage("Transmitting to server failed. Storing the paused job with id: " + job.Id + " to hdd (" + e.ToString() + ")");
-          }
-          finally {
-            KillAppDomain(job.Id); // kill app-domain in every case          
-          }
-        }
-      }
-    }
-
-    /// <summary>
-    /// aborts all running jobs, no results are sent back
-    /// </summary>
-    private void DoAbortAll() {
-      List<Guid> jobIds;
-      lock (executors) {
-        jobIds = new List<Guid>(executors.Keys);
-      }
-      foreach (Guid jobId in jobIds) {
-        KillAppDomain(jobId);
+        }
       }
       clientCom.LogMessage("Aborted all jobs!");
@@ -294,12 +294,10 @@
       clientCom.LogMessage("Pause all received");
 
-      //copy guids because there will be removed items from 'Jobs'
-      List<Guid> jobIds;
-      lock (executors) {
-        jobIds = new List<Guid>(Executors.Keys);
-      }
-
-      foreach (Guid jobId in jobIds) {
-        DoPauseJob(jobId);
+      lock (slaveJobs) {
+        foreach (SlaveJob sj in slaveJobs.Values) {
+          if (!sj.Finished) {
+            sj.PauseJob();
+          }
+        }
       }
     }
@@ -311,12 +309,10 @@
       clientCom.LogMessage("Stop all received");
 
-      //copy guids because there will be removed items from 'Jobs'
-      List<Guid> jobIds;
-      lock (executors) {
-        jobIds = new List<Guid>(executors.Keys);
-      }
-
-      foreach (Guid jobId in jobIds) {
-        DoStopJob(jobId);
+      lock (slaveJobs) {
+        foreach (SlaveJob sj in slaveJobs.Values) {
+          if (!sj.Finished) {
+            sj.StopJob();
+          }
+        }
       }
     }
@@ -341,13 +337,6 @@
       clientCom.LogMessage("Logging out");
 
-
-      lock (executors) {
-        clientCom.LogMessage("executors locked");
-        foreach (KeyValuePair<Guid, AppDomain> kvp in appDomains) {
-          clientCom.LogMessage("Shutting down Appdomain for " + kvp.Key);
-          appDomains[kvp.Key].UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
-          AppDomain.Unload(kvp.Value);
-        }
-      }
+      DoAbortAll();
+
       WcfService.Instance.Disconnect();
       clientCom.Shutdown();
@@ -382,159 +371,4 @@
 
     /// <summary>
-    /// Pauses a job, which means sending it to the server and killing it locally;
-    /// atm only used when executor is waiting for child jobs
-    /// </summary>
-    public void PauseWaitJob(JobData data) {
-      if (!Executors.ContainsKey(data.JobId)) {
-        clientCom.LogMessage("Can't find job with id " + data.JobId);
-      } else {
-        Job job = wcfService.GetJob(data.JobId);
-        wcfService.UpdateJobData(job, data, ConfigManager.Instance.GetClientInfo().Id, JobState.Paused);
-        wcfService.UpdateJobState(job.Id, JobState.Waiting, null);
-      }
-      KillAppDomain(data.JobId);
-    }
-
-    /// <summary>
-    /// serializes the finished job and submits it to the server. If, at the time, a network connection is unavailable, the Job gets stored on the disk. 
-    /// once the connection gets reestablished, the job gets submitted
-    /// </summary>
-    public void SendFinishedJob(Guid jobId) {
-      try {
-        clientCom.LogMessage("Getting the finished job with id: " + jobId);
-        if (!executors.ContainsKey(jobId)) {
-          clientCom.LogMessage("Executor doesn't exist");
-          return;
-        }
-        if (!executors.ContainsKey(jobId)) {
-          clientCom.LogMessage("Job doesn't exist");
-          return;
-        }
-        Job job = wcfService.GetJob(jobId);
-        job.ExecutionTime = executors[jobId].ExecutionTime;
-
-        if (executors[jobId].Aborted) {
-          SlaveStatusInfo.JobsAborted++;
-        } else {
-          SlaveStatusInfo.JobsProcessed++;
-        }
-
-        if (executors[jobId].CurrentException != string.Empty) {
-          wcfService.UpdateJobState(jobId, JobState.Failed, executors[jobId].CurrentException);
-        }
-
-        JobData sJob = executors[jobId].GetFinishedJob();
-        try {
-          clientCom.LogMessage("Sending the finished job with id: " + jobId);
-          wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Finished);
-        }
-        catch (Exception e) {
-          clientCom.LogMessage("Transmitting to server failed. Storing the finished job with id: " + jobId + " to hdd (" + e.ToString() + ")");
-        }
-        finally {
-          KillAppDomain(jobId);
-          heartbeatManager.AwakeHeartBeatThread();
-        }
-      }
-      catch (Exception e) {
-        OnExceptionOccured(e);
-      }
-    }
-
-    private static object startInAppDomainLocker = new object();
-    private static Mutex startInAppDomainMutex = new Mutex();
-    /// <summary>
-    /// A new Job from the wcfService has been received and will be started within a AppDomain.
-    /// </summary>    
-    private void StartJobInAppDomain(Job job, JobData jobData) {
-      clientCom.LogMessage("Received new job with id " + job.Id);
-      clientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
-
-      startInAppDomainMutex.WaitOne(); // mutex is used instead of lock to be able to release it before executor.Start() is called (which may take long time)
-      bool released = false;
-
-        if (executors.ContainsKey(job.Id)) {
-          clientCom.LogMessage("Job with key " + job.Id + " already exists. Job will be ignored.");
-          return;
-        }
-
-        String pluginDir = Path.Combine(PluginCache.Instance.PluginTempBaseDir, job.Id.ToString());
-        bool pluginsPrepared = false;
-        string configFileName = string.Empty;
-
-        try {
-          PluginCache.Instance.PreparePlugins(job, out configFileName);
-          clientCom.LogMessage("Plugins fetched for job " + job.Id);
-          pluginsPrepared = true;
-        }
-        catch (Exception exception) {
-          clientCom.LogMessage(string.Format("Copying plugins for job {0} failed: {1}", job.Id, exception));
-          wcfService.UpdateJobState(job.Id, JobState.Failed, exception.ToString());
-          SlaveStatusInfo.JobsAborted++;
-        }
-
-        if (pluginsPrepared) {
-          try {
-            //TODO: switch back to unprivileged sandbox
-            AppDomain appDomain = HeuristicLab.PluginInfrastructure.Sandboxing.SandboxManager.CreateAndInitPrivilegedSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
-            appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
-            Executor executor;
-            appDomains.Add(job.Id, appDomain);
-            clientCom.LogMessage("Creating AppDomain");
-            executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
-            clientCom.LogMessage("Created AppDomain");
-            executor.Core = this;
-            executor.JobId = job.Id;
-            executor.CoresNeeded = job.CoresNeeded;
-            executor.MemoryNeeded = job.MemoryNeeded;
-            clientCom.LogMessage("Starting Executor for job " + job.Id);
-            lock (executors) {
-              if (executors.ContainsKey(job.Id)) {
-                throw new JobAlreadyExistsException(job.Id);
-              }
-              executors.Add(job.Id, executor);
-            }
-            startInAppDomainMutex.ReleaseMutex();
-            released = true;
-            semaphores[job.Id] = new Semaphore(0, 1);
-            executor.Start(jobData.Data);
-            semaphores[job.Id].Release();
-          }
-          catch (JobAlreadyExistsException e) {
-            clientCom.LogMessage(string.Format("Job {0} has already been started. Job will be ignored", e.JobId));
-          }
-          catch (Exception exception) {
-            clientCom.LogMessage("Creating the Appdomain and loading the job failed for job " + job.Id);
-            clientCom.LogMessage("Error thrown is: " + exception.ToString());
-
-            if (executors.ContainsKey(job.Id) && executors[job.Id].CurrentException != string.Empty) {
-              wcfService.UpdateJobState(job.Id, JobState.Failed, executors[job.Id].CurrentException);
-            } else {
-              wcfService.UpdateJobState(job.Id, JobState.Failed, exception.ToString());
-            }
-            SlaveStatusInfo.JobsAborted++;
-
-            KillAppDomain(job.Id);
-          }
-
-          if (!released)
-            startInAppDomainMutex.ReleaseMutex();
-        }
-      heartbeatManager.AwakeHeartBeatThread();
-    }
-
-    public event EventHandler<EventArgs<Exception>> ExceptionOccured;
-    private void OnExceptionOccured(Exception e) {
-      clientCom.LogMessage("Error: " + e.ToString());
-      var handler = ExceptionOccured;
-      if (handler != null) handler(this, new EventArgs<Exception>(e));
-    }
-
-    private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
-      clientCom.LogMessage("Exception in AppDomain: " + e.ExceptionObject.ToString());
-      KillAppDomain(new Guid(e.ExceptionObject.ToString()));
-    }
-
-    /// <summary>
     /// Enqueues messages from the executor to the message queue. 
     /// This is necessary if the core thread has to execute certain actions, e.g.
@@ -549,64 +383,9 @@
     }
 
-    /// <summary>
-    /// Kill a appdomain with a specific id.
-    /// </summary>
-    /// <param name="jobId">the GUID of the job</param>    
-    public void KillAppDomain(Guid jobId) {
-      if (Thread.CurrentThread.ManagedThreadId != this.coreThreadId) {
-        EnqueueExecutorMessage<Guid>(KillAppDomain, jobId);
-        return;
-      }
-
-      clientCom.LogMessage("Shutting down Appdomain for Job " + jobId);
-      lock (executors) {
-        try {
-          if (executors.ContainsKey(jobId)) {
-            executors[jobId].Dispose();
-            executors.Remove(jobId);
-          }
-
-          if (appDomains.ContainsKey(jobId)) {
-            appDomains[jobId].UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
-            int repeat = 5;
-            while (repeat > 0) {
-              try {
-                semaphores[jobId].WaitOne();
-                AppDomain.Unload(appDomains[jobId]);
-                semaphores[jobId].Dispose();
-                semaphores.Remove(jobId);
-                repeat = 0;
-              }
-              catch (CannotUnloadAppDomainException) {
-                clientCom.LogMessage("Could not unload AppDomain, will try again in 1 sec.");
-                Thread.Sleep(1000);
-                repeat--;
-                if (repeat == 0) {
-                  clientCom.LogMessage("Could not unload AppDomain, shutting down core...");
-                  throw; // rethrow and let app crash
-                }
-              }
-            }
-            appDomains.Remove(jobId);
-          }
-
-          PluginCache.Instance.DeletePluginsForJob(jobId);
-          GC.Collect();
-        }
-        catch (Exception ex) {
-          clientCom.LogMessage("Exception when unloading the appdomain: " + ex.ToString());
-        }
-      }
-      GC.Collect();
-      clientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
-    }
-
-    public override object InitializeLifetimeService() {
-      return null; // avoid destruction of proxy object after 5 minutes
-    }
-
-    public int GetCoresNeeded() {
-      lock (executors) {
-        return executors.Sum(x => x.Value.CoresNeeded);
+    public void RemoveSlaveJobFromList(Guid jobId) {
+      lock (slaveJobs) {
+        if (slaveJobs.ContainsKey(jobId)) {
+          slaveJobs.Remove(jobId);
+        }
       }
     }
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs	(revision 6203)
@@ -40,4 +40,8 @@
     private Semaphore pauseStopSem = new Semaphore(0, 1);
     private Semaphore startJobSem = new Semaphore(0, 1);
+    //make pause or stop wait until start is finished
+    private Semaphore jobStartedSem = new Semaphore(0, 1);
+
+    public ExecutorQueue executorQueue;
 
     public bool SendHeartbeatForExecutor { get; set; }
@@ -72,4 +76,5 @@
     public Executor() {
       SendHeartbeatForExecutor = true;
+      executorQueue = new ExecutorQueue();
     }
 
@@ -89,7 +94,8 @@
         } else {
           Job.Start();
-          if (!startJobSem.WaitOne(TimeSpan.FromSeconds(15))) {
+          if (!jobStartedSem.WaitOne(TimeSpan.FromSeconds(15))) {
             throw new TimeoutException("Timeout when starting the job. JobStarted event was not fired.");
           }
+          jobStartedSem.Release();
         }
       }
@@ -102,7 +108,9 @@
     public void Pause() {
       SendHeartbeatForExecutor = false;
+      // wait until job is started. if this does not happen, the Job is null an we give up
+      jobStartedSem.WaitOne(TimeSpan.FromSeconds(15));
       if (Job == null) {
         currentException = new Exception("Pausing job " + this.JobId + ": Job is null");
-        Core.EnqueueExecutorMessage(Core.KillAppDomain, JobId);
+        return;
       }
 
@@ -121,7 +129,8 @@
     public void Stop() {
       SendHeartbeatForExecutor = false;
+      // wait until job is started. if this does not happen, the Job is null an we give up
+      jobStartedSem.WaitOne(TimeSpan.FromSeconds(15));
       if (Job == null) {
         currentException = new Exception("Stopping job " + this.JobId + ": Job is null");
-        Core.EnqueueExecutorMessage(Core.KillAppDomain, JobId);
       }
       wasJobAborted = true;
@@ -176,6 +185,9 @@
       childJob.PluginsNeededIds = FindPluginsNeeded(e.Value);
 
-      //TODO: is return value needed?
-      WcfService.Instance.AddChildJob(this.JobId, childJob, childJobData);
+      ExecutorMessage msg = new ExecutorMessage(ExecutorMessageType.NewChildJob);
+      msg.MsgData = childJobData;
+      msg.MsgJob = childJob;
+
+      executorQueue.AddMessage(msg);
     }
 
@@ -188,9 +200,11 @@
       jdata.JobId = this.JobId;
 
-      Core.PauseWaitJob(jdata);
+      ExecutorMessage msg = new ExecutorMessage(ExecutorMessageType.WaitForChildJobs);
+      msg.MsgData = jdata;
+      executorQueue.AddMessage(msg);
     }
 
     private void Job_DeleteChildJobs(object sender, EventArgs e) {
-      WcfService.Instance.DeleteChildJobs(JobId);
+      executorQueue.AddMessage(ExecutorMessageType.DeleteChildJobs);
     }
 
@@ -198,6 +212,7 @@
       HeuristicLab.Common.EventArgs<Exception> ex = (HeuristicLab.Common.EventArgs<Exception>)e;
       currentException = ex.Value;
-      Core.EnqueueExecutorMessage(Core.SendFinishedJob, JobId);
       Aborted = true;
+
+      executorQueue.AddMessage(ExecutorMessageType.JobFailed);
     }
 
@@ -208,5 +223,5 @@
       } else {
         //it's a clean and finished job, so send it
-        Core.EnqueueExecutorMessage(Core.SendFinishedJob, JobId);
+        executorQueue.AddMessage(ExecutorMessageType.JobStopped);
       }
     }
@@ -216,6 +231,6 @@
         if (currentException == null) {
           currentException = new Exception("Getting finished job " + this.JobId + ": Job is null");
-        }
-        Core.EnqueueExecutorMessage(Core.KillAppDomain, JobId);
+          return GetJob();
+        }
       }
 
@@ -234,5 +249,4 @@
     }
 
-
     public JobData GetPausedJob() {
       if (Job.ExecutionState != HeuristicLab.Core.ExecutionState.Paused) {
@@ -247,5 +261,5 @@
 
     void Job_JobStarted(object sender, EventArgs e) {
-      startJobSem.Release();
+      jobStartedSem.Release();
     }
 
@@ -255,5 +269,13 @@
       } else {
         JobData jdata = new JobData();
-        jdata.Data = PersistenceUtil.Serialize(Job);
+        if (Job == null) {
+          //send empty job and save exception
+          jdata.Data = PersistenceUtil.Serialize(new JobData());
+          if (currentException == null) {
+            currentException = new Exception("Job with id " + this.JobId + " is null, sending empty job");
+          }
+        } else {
+          jdata.Data = PersistenceUtil.Serialize(Job);
+        }
         jdata.JobId = JobId;
         return jdata;
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ExecutorMessage.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ExecutorMessage.cs	(revision 6203)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ExecutorMessage.cs	(revision 6203)
@@ -0,0 +1,25 @@
+﻿
+using System;
+namespace HeuristicLab.Clients.Hive.SlaveCore {
+  [Serializable]
+  public enum ExecutorMessageType {
+    JobStopped,
+    JobFailed,
+    NewChildJob,
+    WaitForChildJobs,
+    DeleteChildJobs,
+    StopExecutorMonitoringThread
+  }
+
+  [Serializable]
+  public class ExecutorMessage {
+    public ExecutorMessageType MessageType { get; set; }
+
+    public ExecutorMessage(ExecutorMessageType messageType) {
+      MessageType = messageType;
+    }
+
+    public JobData MsgData { get; set; }
+    public Job MsgJob { get; set; }
+  }
+}
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ExecutorQueue.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ExecutorQueue.cs	(revision 6203)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ExecutorQueue.cs	(revision 6203)
@@ -0,0 +1,56 @@
+﻿
+using System;
+using System.Collections.Generic;
+using System.Threading;
+namespace HeuristicLab.Clients.Hive.SlaveCore {
+
+  public class ExecutorQueue : MarshalByRefObject {
+    private Queue<ExecutorMessage> queue = null;
+    private Semaphore semaphore = null;
+
+    /// <summary>
+    /// Creates a new MessageQueue Object.
+    /// A new Queue and a Semaphore is created. The Semaphore is set to a max size of 5000.
+    /// </summary>
+    public ExecutorQueue() {
+      queue = new Queue<ExecutorMessage>();
+      semaphore = new Semaphore(0, 5000);
+    }
+
+    /// <summary>
+    /// Returns the oldest ExecutorMessage Object from the Queue. 
+    /// </summary>
+    /// <returns>the oldest ExecutorMessage Object</returns>
+    public ExecutorMessage GetMessage() {
+      semaphore.WaitOne();
+      lock (this) {
+        if (queue.Count > 0) {
+          return queue.Dequeue();
+        }
+      }
+      return null;
+    }
+
+    /// <summary>
+    /// Adds a ExecutorMessage Object to the Queue 
+    /// </summary>
+    /// <param name="message">the ExecutorMessage</param>
+    public void AddMessage(ExecutorMessage message) {
+      lock (this) {
+        queue.Enqueue(message);
+        semaphore.Release();
+      }
+    }
+
+    /// <summary>
+    /// Adds a message to the Queue. The ExecutorMessage Object is built in the Method
+    /// </summary>
+    /// <param name="message">the Message</param>
+    public void AddMessage(ExecutorMessageType message) {
+      lock (this) {
+        queue.Enqueue(new ExecutorMessage(message));
+        semaphore.Release();
+      }
+    }
+  }
+}
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeartbeatManager.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeartbeatManager.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeartbeatManager.cs	(revision 6203)
@@ -89,5 +89,5 @@
               Heartbeat heartBeatData = new Heartbeat {
                 SlaveId = info.Id,
-                FreeCores = info.Cores.HasValue ? info.Cores.Value - ConfigManager.Instance.GetUsedCores() : 0,
+                FreeCores = info.Cores.HasValue ? info.Cores.Value - SlaveStatusInfo.UsedCores : 0,
                 FreeMemory = ConfigManager.GetFreeMemory(),
                 JobProgress = ConfigManager.Instance.GetExecutionTimeOfAllJobs(),
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeuristicLab.Clients.Hive.Slave-3.4.csproj
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeuristicLab.Clients.Hive.Slave-3.4.csproj	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeuristicLab.Clients.Hive.Slave-3.4.csproj	(revision 6203)
@@ -98,7 +98,9 @@
   <ItemGroup>
     <Compile Include="ConfigManager.cs" />
+    <Compile Include="Exceptions\JobAlreadyExistsException.cs" />
     <Compile Include="Exceptions\JobNotFoundException.cs" />
     <Compile Include="Exceptions\JobNotDataFoundException.cs" />
-    <Compile Include="Exceptions\JobAlreadyExistsException.cs" />
+    <Compile Include="ExecutorMessage.cs" />
+    <Compile Include="ExecutorQueue.cs" />
     <Compile Include="SlaveClientCom.cs" />
     <Compile Include="Core.cs" />
@@ -120,4 +122,5 @@
     <Compile Include="ServiceContracts\ISlaveCommunicationCallbacks.cs" />
     <Compile Include="SlaveCommunicationService.cs" />
+    <Compile Include="SlaveJob.cs" />
     <Compile Include="SlaveStatusInfo.cs" />
     <Compile Include="StatusCommons.cs" />
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/MessageQueue.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/MessageQueue.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/MessageQueue.cs	(revision 6203)
@@ -52,7 +52,7 @@
     /// </summary>
     /// <returns>nothing</returns>
-    public override object InitializeLifetimeService() {
+    /*public override object InitializeLifetimeService() {
       return null;
-    }
+    } */
 
     /// <summary>
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveJob.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveJob.cs	(revision 6203)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveJob.cs	(revision 6203)
@@ -0,0 +1,356 @@
+﻿
+using System;
+using System.IO;
+using System.Threading;
+using HeuristicLab.Clients.Hive.SlaveCore.ServiceContracts;
+
+namespace HeuristicLab.Clients.Hive.SlaveCore {
+
+  public class SlaveJob : MarshalByRefObject {
+    private Executor executor;
+    private AppDomain appDomain;
+    private Semaphore waitForStartBeforeKillSem;
+    private bool executorMonitoringRun;
+    private Thread executorMonitoringThread;
+    private Core core;
+    private bool finished;
+    private int coresNeeded;
+
+    private ISlaveCommunication clientCom;
+    private WcfService wcfService;
+
+    public Guid JobId;
+    public Executor JobExecutor { get { return executor; } }
+    public bool Finished { get { return finished; } }
+
+
+    public SlaveJob(Core core) {
+      clientCom = SlaveClientCom.Instance.ClientCom;
+      wcfService = WcfService.Instance;
+      waitForStartBeforeKillSem = new Semaphore(0, 1);
+      executorMonitoringRun = true;
+      this.core = core;
+      finished = false;
+    }
+
+    public void CalculateJob(Guid jobId) {
+      Job job = wcfService.GetJob(jobId);
+      if (job == null) throw new JobNotFoundException(jobId);
+      coresNeeded = job.CoresNeeded;
+      SlaveStatusInfo.IncrementUsedCores(coresNeeded);
+
+      JobData jobData = wcfService.GetJobData(job.Id);
+      if (jobData == null) throw new JobDataNotFoundException(jobId);
+      SlaveStatusInfo.IncrementJobsFetched();
+      job = wcfService.UpdateJobState(job.Id, JobState.Calculating, null);
+      if (job == null) throw new JobNotFoundException(jobId);
+      StartJobInAppDomain(job, jobData);
+    }
+
+    public void StopJob() {
+      if (executor == null) {
+        clientCom.LogMessage(string.Format("StopJob: job with id {0} is missing the executor", JobId));
+      } else {
+        Job job = wcfService.GetJob(JobId);
+
+        if (job != null) {
+          executor.Stop();
+
+          try {
+            JobData sJob = executor.GetFinishedJob();
+            job.ExecutionTime = executor.ExecutionTime;
+
+            if (executor.CurrentException != string.Empty) {
+              wcfService.UpdateJobState(job.Id, JobState.Failed, executor.CurrentException);
+            }
+            SlaveStatusInfo.IncrementJobsAborted();
+
+            clientCom.LogMessage("Sending the stopped job with id: " + job.Id);
+            wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Aborted);
+          }
+          catch (Exception e) {
+            clientCom.LogMessage("Transmitting to server failed. Storing the paused job with id: " + job.Id + " to hdd (" + e.ToString() + ")");
+          }
+          finally {
+            KillAppDomain(); // kill appdomain in every case          
+          }
+        }
+      }
+    }
+
+    public void PauseJob() {
+      if (executor == null) {
+        clientCom.LogMessage("PauseJob: Can't pause job with uninitialized executor");
+      } else {
+        Job job = wcfService.GetJob(JobId);
+
+        if (job != null) {
+          executor.Pause();
+          JobData sJob = executor.GetPausedJob();
+          job.ExecutionTime = executor.ExecutionTime;
+
+          try {
+            if (executor.CurrentException != string.Empty) {
+              wcfService.UpdateJobState(job.Id, JobState.Failed, executor.CurrentException);
+              SlaveStatusInfo.IncrementJobsFailed();
+            } else {
+              SlaveStatusInfo.IncrementJobsFinished();
+            }
+            clientCom.LogMessage("Sending the paused job with id: " + job.Id);
+            wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Paused);
+          }
+          catch (Exception e) {
+            clientCom.LogMessage("Transmitting to server failed. Storing the paused job with id: " + job.Id + " to hdd (" + e.ToString() + ")");
+          }
+          finally {
+            KillAppDomain(); // kill appdomain in every case          
+          }
+        }
+      }
+    }
+
+    /// <summary>
+    /// Pauses a job, which means sending it to the server and killing it locally;
+    /// atm only used when executor is waiting for child jobs
+    /// </summary>
+    public void PauseWaitJob(JobData data) {
+      try {
+        if (executor == null) {
+          clientCom.LogMessage(string.Format("PauseWaitJob: Can't pause job with id {0} with uninitialized executor", JobId));
+        } else {
+          Job job = wcfService.GetJob(data.JobId);
+          wcfService.UpdateJobData(job, data, ConfigManager.Instance.GetClientInfo().Id, JobState.Paused);
+          wcfService.UpdateJobState(job.Id, JobState.Waiting, null);
+        }
+      }
+      catch (Exception ex) {
+        clientCom.LogMessage(string.Format("Pausing job with id {0} failed. Exception: {1}", JobId, ex.ToString()));
+      }
+      finally {
+        KillAppDomain();
+      }
+    }
+
+    /// <summary>
+    /// serializes the finished job and submits it to the server. If, at the time, a network connection is unavailable, the Job gets stored on the disk. 
+    /// once the connection gets reestablished, the job gets submitted
+    /// </summary>
+    public void SendFinishedJob() {
+      try {
+        clientCom.LogMessage("Getting the finished job with id: " + JobId);
+        if (executor == null) {
+          clientCom.LogMessage(string.Format("SendFinishedJob: Can't pause job with id {0} with uninitialized executor", JobId));
+          return;
+        }
+
+        Job job = wcfService.GetJob(JobId);
+        job.ExecutionTime = executor.ExecutionTime;
+
+        if (executor.Aborted) {
+          SlaveStatusInfo.IncrementJobsAborted();
+        } else {
+          SlaveStatusInfo.IncrementJobsFinished();
+        }
+
+        if (executor.CurrentException != string.Empty) {
+          wcfService.UpdateJobState(JobId, JobState.Failed, executor.CurrentException);
+        }
+
+        try {
+          JobData sJob = executor.GetFinishedJob();
+          clientCom.LogMessage("Sending the finished job with id: " + JobId);
+          wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Finished);
+        }
+        catch (Exception e) {
+          clientCom.LogMessage("Transmitting to server failed. Storing the finished job with id: " + JobId + " to hdd (" + e.ToString() + ")");
+        }
+        finally {
+          KillAppDomain();
+          Core.HBManager.AwakeHeartBeatThread();
+        }
+      }
+      catch (Exception e) {
+        clientCom.LogMessage("SendFinishedJob: The following exception has been thrown: " + e.ToString());
+      }
+    }
+
+    /// <summary>
+    /// A new Job from the wcfService has been received and will be started within a AppDomain.
+    /// </summary>    
+    private void StartJobInAppDomain(Job job, JobData jobData) {
+      JobId = job.Id;
+
+      clientCom.LogMessage("Received new job with id " + job.Id);
+      clientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
+
+      String pluginDir = Path.Combine(PluginCache.Instance.PluginTempBaseDir, job.Id.ToString());
+      bool pluginsPrepared = false;
+      string configFileName = string.Empty;
+
+      try {
+        PluginCache.Instance.PreparePlugins(job, out configFileName);
+        clientCom.LogMessage("Plugins fetched for job " + job.Id);
+        pluginsPrepared = true;
+      }
+      catch (Exception exception) {
+        clientCom.LogMessage(string.Format("Copying plugins for job {0} failed: {1}", job.Id, exception));
+        wcfService.UpdateJobState(job.Id, JobState.Failed, exception.ToString());
+        SlaveStatusInfo.IncrementJobsFailed();
+        core.RemoveSlaveJobFromList(JobId);
+        finished = true;
+        SlaveStatusInfo.DecrementUsedCores(coresNeeded);
+      }
+
+      if (pluginsPrepared) {
+        try {
+          appDomain = HeuristicLab.PluginInfrastructure.Sandboxing.SandboxManager.CreateAndInitSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
+          appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
+
+          clientCom.LogMessage("Creating AppDomain");
+          executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
+          clientCom.LogMessage("Created AppDomain");
+
+          executor.JobId = job.Id;
+          executor.CoresNeeded = job.CoresNeeded;
+          executor.MemoryNeeded = job.MemoryNeeded;
+          clientCom.LogMessage("Starting Executor for job " + job.Id);
+
+          executor.Start(jobData.Data);
+          waitForStartBeforeKillSem.Release();
+
+          StartExecutorMonitoringThread();
+        }
+        catch (Exception exception) {
+          clientCom.LogMessage("Creating the Appdomain and loading the job failed for job " + job.Id);
+          clientCom.LogMessage("Error thrown is: " + exception.ToString());
+
+          if (executor != null && executor.CurrentException != string.Empty) {
+            wcfService.UpdateJobState(job.Id, JobState.Failed, executor.CurrentException);
+          } else {
+            wcfService.UpdateJobState(job.Id, JobState.Failed, exception.ToString());
+          }
+          SlaveStatusInfo.IncrementJobsFailed();
+
+          KillAppDomain();
+        }
+      }
+      Core.HBManager.AwakeHeartBeatThread();
+    }
+
+    /// <summary>
+    /// Kill a appdomain with a specific id.
+    /// </summary>
+    /// <param name="JobId">the GUID of the job</param>    
+    public void KillAppDomain() {
+      clientCom.LogMessage("Shutting down Appdomain for Job " + JobId);
+
+      try {
+        StopExecutorMonitoringThread();
+        finished = true;
+        SlaveStatusInfo.DecrementUsedCores(coresNeeded);
+
+        if (executor != null) {
+          executor.Dispose();
+        }
+
+        if (appDomain != null) {
+          appDomain.UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
+          int repeat = 5;
+          while (repeat > 0) {
+            try {
+              waitForStartBeforeKillSem.WaitOne();
+              AppDomain.Unload(appDomain);
+              waitForStartBeforeKillSem.Dispose();
+              repeat = 0;
+            }
+            catch (CannotUnloadAppDomainException) {
+              clientCom.LogMessage("Could not unload AppDomain, will try again in 1 sec.");
+              Thread.Sleep(1000);
+              repeat--;
+              if (repeat == 0) {
+                clientCom.LogMessage("Could not unload AppDomain, shutting down core...");
+                throw; // rethrow and let app crash
+              }
+            }
+          }
+        }
+
+        PluginCache.Instance.DeletePluginsForJob(JobId);
+        GC.Collect();
+      }
+      catch (Exception ex) {
+        clientCom.LogMessage("Exception when unloading the appdomain: " + ex.ToString());
+      }
+      finally {
+        core.RemoveSlaveJobFromList(JobId);
+      }
+
+      GC.Collect();
+      clientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
+    }
+
+    private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
+      clientCom.LogMessage("Exception in AppDomain: " + e.ExceptionObject.ToString());
+      KillAppDomain();
+    }
+
+
+    #region ExecutorMonitorThread
+
+    private void StartExecutorMonitoringThread() {
+      executorMonitoringThread = new Thread(MonitorExecutor);
+      executorMonitoringThread.Start();
+    }
+
+    private void StopExecutorMonitoringThread() {
+      if (executorMonitoringThread != null) {
+        if (executorMonitoringRun) {
+          executorMonitoringRun = false;
+          executor.executorQueue.AddMessage(ExecutorMessageType.StopExecutorMonitoringThread);
+        }
+      }
+    }
+
+    /// <summary>
+    /// Because the executor is in an appdomain and is not able to call back 
+    /// (because of security -> lease time for marshall-by-ref object is 5 min),
+    /// we have to poll the executor for events we have to react to (e.g. job finished...)    
+    /// </summary>
+    private void MonitorExecutor() {
+      while (executorMonitoringRun) {
+        //this blocks through the appdomain border, that's why the lease gets renewed
+        ExecutorMessage message = executor.executorQueue.GetMessage();
+
+        switch (message.MessageType) {
+          case ExecutorMessageType.JobStopped:
+            executorMonitoringRun = false;
+            SendFinishedJob();
+            break;
+
+          case ExecutorMessageType.JobFailed:
+            executorMonitoringRun = false;
+            SendFinishedJob();
+            break;
+
+          case ExecutorMessageType.NewChildJob:
+            WcfService.Instance.AddChildJob(JobId, message.MsgJob, message.MsgData);
+            break;
+
+          case ExecutorMessageType.WaitForChildJobs:
+            executorMonitoringRun = false;
+            PauseWaitJob(message.MsgData);
+            break;
+
+          case ExecutorMessageType.DeleteChildJobs:
+            WcfService.Instance.DeleteChildJobs(JobId);
+            break;
+
+          case ExecutorMessageType.StopExecutorMonitoringThread:
+            executorMonitoringRun = false;
+            return;
+        }
+      }
+    }
+    #endregion
+  }
+}
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveStatusInfo.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveStatusInfo.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveStatusInfo.cs	(revision 6203)
@@ -24,8 +24,89 @@
 namespace HeuristicLab.Clients.Hive.SlaveCore {
   public class SlaveStatusInfo {
-    static public int JobsProcessed { get; set; }
-    static public int JobsAborted { get; set; }
-    static public int JobsFetched { get; set; }
+    static private int jobsFinished; // everything went fine
+    static private int jobsAborted;  // server sent stop or abort
+    static private int jobsFetched;  // number of fetched jobs
+    static private int jobsFailed;   // jobs that failed in the sandbox
+    static private int usedCores;    // number of cores currently used
+
     static public DateTime LoginTime { get; set; }
+    static private Object jobStatLock = new Object();
+    static private Object coreLock = new Object();
+
+    static public int UsedCores {
+      get {
+        lock (coreLock) {
+          return usedCores;
+        }
+      }
+    }
+
+    static public int JobsFinished {
+      get {
+        lock (jobStatLock) {
+          return jobsFinished;
+        }
+      }
+    }
+
+    static public int JobsAborted {
+      get {
+        lock (jobStatLock) {
+          return jobsAborted;
+        }
+      }
+    }
+
+    static public int JobsFetched {
+      get {
+        lock (jobStatLock) {
+          return jobsFetched;
+        }
+      }
+    }
+
+    static public int JobsFailed {
+      get {
+        lock (jobStatLock) {
+          return jobsFailed;
+        }
+      }
+    }
+
+    public static void IncrementJobsFinished() {
+      lock (jobStatLock) {
+        jobsFinished++;
+      }
+    }
+
+    public static void IncrementJobsFailed() {
+      lock (jobStatLock) {
+        jobsFailed++;
+      }
+    }
+
+    public static void IncrementJobsAborted() {
+      lock (jobStatLock) {
+        jobsAborted++;
+      }
+    }
+
+    public static void IncrementJobsFetched() {
+      lock (jobStatLock) {
+        jobsFetched++;
+      }
+    }
+
+    public static void IncrementUsedCores(int val) {
+      lock (coreLock) {
+        usedCores += val;
+      }
+    }
+
+    public static void DecrementUsedCores(int val) {
+      lock (coreLock) {
+        usedCores -= val;
+      }
+    }
   }
 }
Index: /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/StatusCommons.cs
===================================================================
--- /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/StatusCommons.cs	(revision 6202)
+++ /branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/StatusCommons.cs	(revision 6203)
@@ -43,13 +43,17 @@
     public int JobsFetched { get; set; }
     [DataMember]
-    public int JobsDone { get; set; }
+    public int JobsFinished { get; set; }
     [DataMember]
     public int JobsAborted { get; set; }
+    [DataMember]
+    public int JobsFailed { get; set; }
     [DataMember]
     public List<JobStatus> Jobs { get; set; }
 
     public override string ToString() {
-      return string.Format("Status: {0}, Fetched/Done/Aborted: {1},{2},{3}", Status, JobsFetched, JobsDone, JobsAborted);
+      return string.Format("Status: {0}, Fetched/Finished/Aborted/Failed: {1},{2},{3},{4}", Status, JobsFetched, JobsFinished, JobsAborted, JobsFailed);
     }
+
+
   }
 }
