[5095] | 1 | using System;
|
---|
| 2 | using System.Linq;
|
---|
| 3 | using HeuristicLab.Services.Hive.Common;
|
---|
| 4 | using HeuristicLab.Services.Hive.Common.DataTransfer;
|
---|
| 5 |
|
---|
| 6 | namespace HeuristicLab.Services.Hive {
|
---|
| 7 | /// <summary>
|
---|
[5593] | 8 | /// This class offers methods for cleaning up offline slaves and jobs
|
---|
[5095] | 9 | /// </summary>
|
---|
| 10 | public class LifecycleManager : ILifecycleManager {
|
---|
| 11 | private DataAccess.IHiveDao dao {
|
---|
| 12 | get { return ServiceLocator.Instance.HiveDao; }
|
---|
| 13 | }
|
---|
| 14 | private HeuristicLab.Services.Hive.DataAccess.TransactionManager trans {
|
---|
| 15 | get { return ServiceLocator.Instance.TransactionManager; }
|
---|
| 16 | }
|
---|
| 17 | private IAuthorizationManager auth {
|
---|
| 18 | get { return ServiceLocator.Instance.AuthorizationManager; }
|
---|
| 19 | }
|
---|
[5593] | 20 | private ILogger log {
|
---|
| 21 | get { return LogFactory.GetLogger(this.GetType().Namespace); }
|
---|
[5095] | 22 | }
|
---|
| 23 |
|
---|
[5593] | 24 | public void Cleanup() {
|
---|
| 25 | log.Log("LifecycleManager.Cleanup()");
|
---|
| 26 | SetTimeoutSlavesOffline();
|
---|
| 27 | FinishParentJobs();
|
---|
[5095] | 28 | }
|
---|
| 29 |
|
---|
| 30 | /// <summary>
|
---|
[5526] | 31 | /// Searches for slaves which are timed out, puts them and their jobs offline
|
---|
| 32 | /// </summary>
|
---|
| 33 | private void SetTimeoutSlavesOffline() {
|
---|
| 34 | var slaves = dao.GetSlaves(x => x.SlaveState != SlaveState.Offline);
|
---|
| 35 | foreach (Slave slave in slaves) {
|
---|
| 36 | if (!slave.LastHeartbeat.HasValue || (DateTime.Now - slave.LastHeartbeat.Value).TotalSeconds > ApplicationConstants.HeartbeatTimeout) {
|
---|
| 37 | slave.SlaveState = SlaveState.Offline;
|
---|
| 38 | SetJobsWaiting(slave.Id);
|
---|
| 39 | dao.UpdateSlave(slave);
|
---|
| 40 | }
|
---|
| 41 | }
|
---|
| 42 | }
|
---|
| 43 |
|
---|
| 44 | /// <summary>
|
---|
| 45 | /// Looks for parent jobs which have FinishWhenChildJobsFinished and set their state to finished
|
---|
| 46 | /// </summary>
|
---|
| 47 | private void FinishParentJobs() {
|
---|
| 48 | var parentJobsToFinish = dao.GetParentJobs(dao.GetResources(x => true).Select(x => x.Id), 0, true);
|
---|
| 49 | foreach (var job in parentJobsToFinish) {
|
---|
[5636] | 50 | dao.UpdateJobState(job.Id, JobState.Finished, null, null, string.Empty);
|
---|
[5526] | 51 | }
|
---|
| 52 | }
|
---|
| 53 |
|
---|
| 54 | private void SetJobsWaiting(Guid slaveId) {
|
---|
| 55 | var jobs = dao.GetJobs(x => x.State == JobState.Calculating).Where(x => x.StateLog.Last().SlaveId == slaveId);
|
---|
[5095] | 56 | foreach (var j in jobs) {
|
---|
[5718] | 57 | dao.UpdateJobState(j.Id, JobState.Waiting, slaveId, null, "Slave timed out");
|
---|
[5095] | 58 | }
|
---|
| 59 | }
|
---|
[5593] | 60 |
|
---|
[5095] | 61 | }
|
---|
| 62 | }
|
---|