Changeset 6112

Timestamp:

05/03/11 18:08:42 (13 years ago)

Author:

ascheibe

Message:

#1233

• HeartbeatManager: don't sleep while starting jobs
• Executor: make Start() blocking
• shutdown properly if an uncaught exception is thrown

Location:

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4

Files:

• Core.cs (modified) (5 diffs)
• Executor.cs (modified) (5 diffs)
• HeartbeatManager.cs (modified) (4 diffs)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs

@@ -100 +100 @@
           //try to log with clientCom. if this works the user sees at least a message, 
           //else an exception will be thrown anyways.
-          clientCom.LogMessage("Error on startup: " + ex.ToString() +
+          clientCom.LogMessage("Uncaught exception: " + ex.ToString() +
             Environment.NewLine + "Core is going to shutdown.");
         }
+        ShutdownCore();
       }
       finally {
@@ -177 +178 @@
               // handle exception of task
               clientCom.LogMessage(t.Exception.ToString());
+              wcfService.UpdateJobState(container.JobId, JobState.Failed, t.Exception.ToString());
+              SlaveStatusInfo.JobsAborted++;
             }, TaskContinuationOptions.OnlyOnFaulted);
             break;
@@ -474 +477 @@
         catch (Exception exception) {
           clientCom.LogMessage(string.Format("Copying plugins for job {0} failed: {1}", myJob.Id, exception));
+          wcfService.UpdateJobState(myJob.Id, JobState.Failed, exception.ToString());
+          SlaveStatusInfo.JobsAborted++;
           lock (executors) {
             if (jobs.ContainsKey(myJob.Id)) {
@@ -493 +498 @@
             executor.Core = this;
             clientCom.LogMessage("Starting Executor for job " + myJob.Id);
-
             executor.Start(jobData.Data);
-            
+
             lock (executors) {
               executors.Add(myJob.Id, executor);
@@ -503 +507 @@
             clientCom.LogMessage("Creating the Appdomain and loading the job failed for job " + myJob.Id);
             clientCom.LogMessage("Error thrown is: " + exception.ToString());
+
+            if (executors.ContainsKey(myJob.Id) && executors[myJob.Id].CurrentException != string.Empty) {
+              wcfService.UpdateJobState(myJob.Id, JobState.Failed, executors[myJob.Id].CurrentException);
+            } else {
+              wcfService.UpdateJobState(myJob.Id, JobState.Failed, exception.ToString());
+            }
+            SlaveStatusInfo.JobsAborted++;
+
             KillAppDomain(myJob.Id);
           }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs

@@ -37 +37 @@
     public Core Core { get; set; }
     private Semaphore pauseStopSem = new Semaphore(0, 1);
+    private Semaphore startJobSem = new Semaphore(0, 1);
 
     public bool SendHeartbeatForExecutor { get; set; }
@@ -86 +87 @@
         } else {
           Job.Start();
-
+          startJobSem.WaitOne();
         }
       }
@@ -140 +141 @@
       Job.DeleteChildJobs += new EventHandler(Job_DeleteChildJobs);
       Job.JobPaused += new EventHandler(Job_JobPaused);
+      Job.JobStarted += new EventHandler(Job_JobStarted);
     }
 
@@ -149 +151 @@
       Job.DeleteChildJobs -= new EventHandler(Job_DeleteChildJobs);
       Job.JobPaused -= new EventHandler(Job_JobPaused);
+      Job.JobStarted -= new EventHandler(Job_JobStarted);
     }
 
@@ -237 +240 @@
     private void Job_JobPaused(object sender, EventArgs e) {
       pauseStopSem.Release();
+    }
+
+    void Job_JobStarted(object sender, EventArgs e) {
+      startJobSem.Release();
     }
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeartbeatManager.cs

@@ -36 +36 @@
     private WcfService wcfService;
     private bool threadStopped;
-    ReaderWriterLockSlim heartBeatThreadIsSleepingLock = new ReaderWriterLockSlim();
 
     public HeartbeatManager() {
@@ -75 +74 @@
 
     private void RunHeartBeatThread() {
-      bool sleepForever;
       while (!threadStopped) {
-        sleepForever = false;
         SlaveClientCom.Instance.ClientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
 
@@ -108 +105 @@
                 msgs.ForEach(mc => SlaveClientCom.Instance.ClientCom.LogMessage(mc.Message.ToString()));
                 msgs.ForEach(mc => MessageQueue.GetInstance().AddMessage(mc));
-                //after fetching a job, we sleep until the core wakes us up!!
-                msgs.ForEach(s => { if (s.Message == MessageContainer.MessageType.CalculateJob) sleepForever = true; });
               }
             }
@@ -118 +113 @@
           OnExceptionOccured(e);
         }
-        if (sleepForever)
-          waitHandle.WaitOne();
-        else
-          waitHandle.WaitOne(this.Interval);
+        waitHandle.WaitOne(this.Interval);
       }
       waitHandle.Close();