1 | using System;
|
---|
2 | using System.Linq;
|
---|
3 | using HeuristicLab.Services.Hive.Common;
|
---|
4 | using HeuristicLab.Services.Hive.Common.DataTransfer;
|
---|
5 |
|
---|
6 | namespace HeuristicLab.Services.Hive {
|
---|
7 | /// <summary>
|
---|
8 | /// This class offers methods for cleaning up offline slaves and jobs
|
---|
9 | /// </summary>
|
---|
10 | public class LifecycleManager : ILifecycleManager {
|
---|
11 | private DataAccess.IHiveDao dao {
|
---|
12 | get { return ServiceLocator.Instance.HiveDao; }
|
---|
13 | }
|
---|
14 | private HeuristicLab.Services.Hive.DataAccess.TransactionManager trans {
|
---|
15 | get { return ServiceLocator.Instance.TransactionManager; }
|
---|
16 | }
|
---|
17 | private IAuthorizationManager auth {
|
---|
18 | get { return ServiceLocator.Instance.AuthorizationManager; }
|
---|
19 | }
|
---|
20 | private ILogger log {
|
---|
21 | get { return LogFactory.GetLogger(this.GetType().Namespace); }
|
---|
22 | }
|
---|
23 |
|
---|
24 | public void Cleanup() {
|
---|
25 | log.Log("LifecycleManager.Cleanup()");
|
---|
26 | SetTimeoutSlavesOffline();
|
---|
27 | FinishParentJobs();
|
---|
28 | }
|
---|
29 |
|
---|
30 | /// <summary>
|
---|
31 | /// Searches for slaves which are timed out, puts them and their jobs offline
|
---|
32 | /// </summary>
|
---|
33 | private void SetTimeoutSlavesOffline() {
|
---|
34 | var slaves = dao.GetSlaves(x => x.SlaveState != SlaveState.Offline);
|
---|
35 | foreach (Slave slave in slaves) {
|
---|
36 | if (!slave.LastHeartbeat.HasValue || (DateTime.Now - slave.LastHeartbeat.Value).TotalSeconds > ApplicationConstants.HeartbeatTimeout) {
|
---|
37 | slave.SlaveState = SlaveState.Offline;
|
---|
38 | SetJobsWaiting(slave.Id);
|
---|
39 | dao.UpdateSlave(slave);
|
---|
40 | }
|
---|
41 | }
|
---|
42 | }
|
---|
43 |
|
---|
44 | /// <summary>
|
---|
45 | /// Looks for parent jobs which have FinishWhenChildJobsFinished and set their state to finished
|
---|
46 | /// </summary>
|
---|
47 | private void FinishParentJobs() {
|
---|
48 | var parentJobsToFinish = dao.GetParentJobs(dao.GetResources(x => true).Select(x => x.Id), 0, true);
|
---|
49 | foreach (var job in parentJobsToFinish) {
|
---|
50 | dao.UpdateJobState(job.Id, JobState.Finished, null, null, string.Empty);
|
---|
51 | }
|
---|
52 | }
|
---|
53 |
|
---|
54 | private void SetJobsWaiting(Guid slaveId) {
|
---|
55 | var jobs = dao.GetJobs(x => x.State == JobState.Calculating).Where(x => x.StateLog.Last().SlaveId == slaveId);
|
---|
56 | foreach (var j in jobs) {
|
---|
57 | dao.UpdateJobState(j.Id, JobState.Waiting, slaveId, null, "Slave timed out");
|
---|
58 | }
|
---|
59 | }
|
---|
60 |
|
---|
61 | }
|
---|
62 | }
|
---|