Changeset 11082 for trunk/sources/HeuristicLab.Clients.Hive.Slave/3.3/Executor.cs

Timestamp:

07/03/14 16:03:14 (10 years ago)

Author:

ascheibe

Message:

#2153

• added a new method HandleStartStopPauseError in Executor to handle error conditions in the same way
• added timeouts for semaphores so that failed tasks or tasks with endless loops don't block the slave
• removed ExceptionOccured events from Executor/SlaveTask/TaskManager and use TaskFailed instead
• removed another ExcpetionOccured event in HeartbeatManager that was never used

File:

• trunk/sources/HeuristicLab.Clients.Hive.Slave/3.3/Executor.cs (modified) (6 diffs)

trunk/sources/HeuristicLab.Clients.Hive.Slave/3.3/Executor.cs

@@ -85 +85 @@
         task.Start();
         if (!startTaskSem.WaitOne(Settings.Default.ExecutorSemTimeouts)) {
-          taskDataInvalid = true;
           throw new TimeoutException("Timeout when starting the task. TaskStarted event was not fired.");
         }
       }
       catch (Exception e) {
-        this.CurrentException = e;
-        taskDataInvalid = true;
-        Task_TaskFailed(this, new EventArgs<Exception>(e));
-      } finally {
+        HandleStartStopPauseError(e);
+      }
+      finally {
         taskStartedSem.Set();
       }
@@ -103 +101 @@
       taskStartedSem.WaitOne(Settings.Default.ExecutorSemTimeouts);
       if (task == null) {
-        CurrentException = new Exception("Pausing task " + this.TaskId + ": Task is null");
-        executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
+        HandleStartStopPauseError(new Exception("Pausing task " + this.TaskId + ": Task is null"));
         return;
       }
@@ -112 +109 @@
           task.Pause();
           //we need to block the pause...
-          pauseStopSem.WaitOne();
+          if (!pauseStopSem.WaitOne(Settings.Default.ExecutorSemTimeouts)) {
+            throw new Exception("Pausing task " + this.TaskId + " timed out.");
+          }
         }
         catch (Exception ex) {
-          CurrentException = new Exception("Error pausing task " + this.TaskId + ": " + ex.ToString());
-          executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
+          HandleStartStopPauseError(ex);
         }
       }
@@ -125 +123 @@
       // wait until task is started. if this does not happen, the Task is null an we give up
       taskStartedSem.WaitOne(Settings.Default.ExecutorSemTimeouts);
+      wasTaskAborted = true;
+
       if (task == null) {
-        CurrentException = new Exception("Stopping task " + this.TaskId + ": Task is null");
-        executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
-      }
-      wasTaskAborted = true;
+        HandleStartStopPauseError(new Exception("Stopping task " + this.TaskId + ": Task is null"));
+        return;
+      }
 
       if ((ExecutionState == ExecutionState.Started) || (ExecutionState == ExecutionState.Paused)) {
         try {
           task.Stop();
-          pauseStopSem.WaitOne();
+          if (!pauseStopSem.WaitOne(Settings.Default.ExecutorSemTimeouts)) {
+            throw new Exception("Stopping task " + this.TaskId + " timed out.");
+          }
         }
         catch (Exception ex) {
-          CurrentException = new Exception("Error stopping task " + this.TaskId + ": " + ex.ToString());
-          executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
+          HandleStartStopPauseError(ex);
         }
       }
@@ -190 +190 @@
       if (task != null && task.ExecutionState == ExecutionState.Started) {
         throw new InvalidStateException("Task is still running");
+      }
+
+      TaskData taskData = null;
+      if (task == null) {
+        if (CurrentException == null) {
+          CurrentException = new Exception("Task with id " + this.TaskId + " is null, sending empty task");
+        }
       } else {
-        TaskData taskData = new TaskData();
-        if (task == null) {
-          //send empty task and save exception
-          taskData.Data = PersistenceUtil.Serialize(new TaskData());
-          if (CurrentException == null) {
-            CurrentException = new Exception("Task with id " + this.TaskId + " is null, sending empty task");
-          }
-        } else {
-          taskData.Data = PersistenceUtil.Serialize(task);
-        }
+        taskData = new TaskData();
+        taskData.Data = PersistenceUtil.Serialize(task);
         taskData.TaskId = TaskId;
-        return taskData;
-      }
+      }
+      return taskData;
     }
 
@@ -211 +210 @@
       task = null;
     }
+
+    private void HandleStartStopPauseError(Exception e) {
+      taskDataInvalid = true;
+      Task_TaskFailed(this, new EventArgs<Exception>(e));
+    }
   }
 }