Ignore:
Timestamp:
07/03/14 16:03:14 (5 years ago)
Author:
ascheibe
Message:

#2153

  • added a new method HandleStartStopPauseError in Executor to handle error conditions in the same way
  • added timeouts for semaphores so that failed tasks or tasks with endless loops don't block the slave
  • removed ExceptionOccured events from Executor/SlaveTask/TaskManager and use TaskFailed instead
  • removed another ExcpetionOccured event in HeartbeatManager that was never used
File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/sources/HeuristicLab.Clients.Hive.Slave/3.3/Executor.cs

    r9456 r11082  
    8585        task.Start();
    8686        if (!startTaskSem.WaitOne(Settings.Default.ExecutorSemTimeouts)) {
    87           taskDataInvalid = true;
    8887          throw new TimeoutException("Timeout when starting the task. TaskStarted event was not fired.");
    8988        }
    9089      }
    9190      catch (Exception e) {
    92         this.CurrentException = e;
    93         taskDataInvalid = true;
    94         Task_TaskFailed(this, new EventArgs<Exception>(e));
    95       } finally {
     91        HandleStartStopPauseError(e);
     92      }
     93      finally {
    9694        taskStartedSem.Set();
    9795      }
     
    103101      taskStartedSem.WaitOne(Settings.Default.ExecutorSemTimeouts);
    104102      if (task == null) {
    105         CurrentException = new Exception("Pausing task " + this.TaskId + ": Task is null");
    106         executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
     103        HandleStartStopPauseError(new Exception("Pausing task " + this.TaskId + ": Task is null"));
    107104        return;
    108105      }
     
    112109          task.Pause();
    113110          //we need to block the pause...
    114           pauseStopSem.WaitOne();
     111          if (!pauseStopSem.WaitOne(Settings.Default.ExecutorSemTimeouts)) {
     112            throw new Exception("Pausing task " + this.TaskId + " timed out.");
     113          }
    115114        }
    116115        catch (Exception ex) {
    117           CurrentException = new Exception("Error pausing task " + this.TaskId + ": " + ex.ToString());
    118           executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
     116          HandleStartStopPauseError(ex);
    119117        }
    120118      }
     
    125123      // wait until task is started. if this does not happen, the Task is null an we give up
    126124      taskStartedSem.WaitOne(Settings.Default.ExecutorSemTimeouts);
     125      wasTaskAborted = true;
     126
    127127      if (task == null) {
    128         CurrentException = new Exception("Stopping task " + this.TaskId + ": Task is null");
    129         executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
    130       }
    131       wasTaskAborted = true;
     128        HandleStartStopPauseError(new Exception("Stopping task " + this.TaskId + ": Task is null"));
     129        return;
     130      }
    132131
    133132      if ((ExecutionState == ExecutionState.Started) || (ExecutionState == ExecutionState.Paused)) {
    134133        try {
    135134          task.Stop();
    136           pauseStopSem.WaitOne();
     135          if (!pauseStopSem.WaitOne(Settings.Default.ExecutorSemTimeouts)) {
     136            throw new Exception("Stopping task " + this.TaskId + " timed out.");
     137          }
    137138        }
    138139        catch (Exception ex) {
    139           CurrentException = new Exception("Error stopping task " + this.TaskId + ": " + ex.ToString());
    140           executorQueue.AddMessage(ExecutorMessageType.ExceptionOccured);
     140          HandleStartStopPauseError(ex);
    141141        }
    142142      }
     
    190190      if (task != null && task.ExecutionState == ExecutionState.Started) {
    191191        throw new InvalidStateException("Task is still running");
     192      }
     193
     194      TaskData taskData = null;
     195      if (task == null) {
     196        if (CurrentException == null) {
     197          CurrentException = new Exception("Task with id " + this.TaskId + " is null, sending empty task");
     198        }
    192199      } else {
    193         TaskData taskData = new TaskData();
    194         if (task == null) {
    195           //send empty task and save exception
    196           taskData.Data = PersistenceUtil.Serialize(new TaskData());
    197           if (CurrentException == null) {
    198             CurrentException = new Exception("Task with id " + this.TaskId + " is null, sending empty task");
    199           }
    200         } else {
    201           taskData.Data = PersistenceUtil.Serialize(task);
    202         }
     200        taskData = new TaskData();
     201        taskData.Data = PersistenceUtil.Serialize(task);
    203202        taskData.TaskId = TaskId;
    204         return taskData;
    205       }
     203      }
     204      return taskData;
    206205    }
    207206
     
    211210      task = null;
    212211    }
     212
     213    private void HandleStartStopPauseError(Exception e) {
     214      taskDataInvalid = true;
     215      Task_TaskFailed(this, new EventArgs<Exception>(e));
     216    }
    213217  }
    214218}
Note: See TracChangeset for help on using the changeset viewer.