Changeset 5786 for branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/LifecycleManager.cs

Timestamp:

03/22/11 11:36:53 (13 years ago)

Author:

cneumuel

Message:

#1233

• implemented correct numbering of BatchRuns
• improvements in ExperimentManager
• fixed bug in server (jobs were scheduled multiple times)
• added exception handling for task in slave
• improved timeout handling of jobs (LifecycleManager)

File:

• branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/LifecycleManager.cs (modified) (4 diffs)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/LifecycleManager.cs

@@ -25 +25 @@
       log.Log("LifecycleManager.Cleanup()");
       SetTimeoutSlavesOffline();
-      FinishParentJobs();
+      SetTimeoutJobsWaiting();
+      FinishParentJobs();      
     }
 
@@ -34 +35 @@
       var slaves = dao.GetSlaves(x => x.SlaveState != SlaveState.Offline);
       foreach (Slave slave in slaves) {
-        if (!slave.LastHeartbeat.HasValue || (DateTime.Now - slave.LastHeartbeat.Value).TotalSeconds > ApplicationConstants.HeartbeatTimeout) {
+        if (!slave.LastHeartbeat.HasValue || (DateTime.Now - slave.LastHeartbeat.Value) > ApplicationConstants.SlaveHeartbeatTimeout) {
           slave.SlaveState = SlaveState.Offline;
           SetJobsWaiting(slave.Id);
@@ -55 +56 @@
       var jobs = dao.GetJobs(x => x.State == JobState.Calculating).Where(x => x.StateLog.Last().SlaveId == slaveId);
       foreach (var j in jobs) {
-        Job job = dao.UpdateJobState(j.Id, JobState.Waiting, slaveId, null, "Slave timed out");
+        Job job = dao.UpdateJobState(j.Id, JobState.Waiting, slaveId, null, "Slave timed out.");
         job.Command = null;
         dao.UpdateJob(job);
@@ -61 +62 @@
     }
 
+    /// <summary>
+    /// Looks for jobs which have not sent heartbeats for some time and reschedules them for calculation
+    /// </summary>
+    private void SetTimeoutJobsWaiting() {
+      var jobs = dao.GetJobs(x => (x.State == JobState.Calculating && (DateTime.Now - x.LastHeartbeat) > ApplicationConstants.CalculatingJobHeartbeatTimeout) 
+                               || (x.State == JobState.Transferring && (DateTime.Now - x.LastHeartbeat) > ApplicationConstants.TransferringJobHeartbeatTimeout));
+      foreach (var j in jobs) {
+        Job job = dao.UpdateJobState(j.Id, JobState.Waiting, null, null, "Slave timed out.");
+        job.Command = null;
+        dao.UpdateJob(job);
+      }
+    }
   }
 }