source: branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/NewEventManager.cs @ 12789

Last change on this file since 12789 was 12789, checked in by dglaser, 7 years ago

#2388:

HeuristicLab.Services.Access.DataAccess-3.3:

  • Added a new method to the TaskDao

HeuristicLab.Services.Hive-3.3:

  • Added NewEventManager.cs
  • Updated HiveJanitor.cs
  • Updated ServiceLocator.cs
File size: 4.2 KB
Line 
1using System;
2using System.Linq;
3using HeuristicLab.Services.Hive.DataAccess;
4using HeuristicLab.Services.Hive.DataAccess.Interfaces;
5
6namespace HeuristicLab.Services.Hive.Manager {
7  public class NewEventManager : IEventManager {
8    private const string SlaveTimeout = "Slave timed out.";
9    private IPersistenceManager PersistenceManager {
10      get { return ServiceLocator.Instance.PersistenceManager; }
11    }
12    private IAuthorizationManager AuthorizationManager {
13      get { return ServiceLocator.Instance.AuthorizationManager; }
14    }
15    private ILogger Log {
16      get { return LogFactory.GetLogger(this.GetType().Namespace); }
17    }
18
19    public void Cleanup() {
20      using (var pm = PersistenceManager) {
21        // same transactions as the old EventManager
22        pm.UseTransaction(() => {
23          SetTimeoutSlavesOffline(pm);
24          SetTimeoutTasksWaiting(pm);
25          DeleteObsoleteSlaves(pm);
26          pm.SubmitChanges();
27        });
28
29        pm.UseTransaction(() => {
30          FinishParentTasks(pm);
31          pm.SubmitChanges();
32        });
33      }
34    }
35
36    /// <summary>
37    /// Searches for slaves which are timed out, puts them and their task offline
38    /// </summary>
39    private void SetTimeoutSlavesOffline(IPersistenceManager pm) {
40      var slaveDao = pm.SlaveDao;
41      var slaves = slaveDao.GetOnlineSlaves();
42      foreach (var slave in slaves) {
43        if (!slave.LastHeartbeat.HasValue ||
44            (DateTime.Now - slave.LastHeartbeat.Value) >
45            HeuristicLab.Services.Hive.Properties.Settings.Default.SlaveHeartbeatTimeout) {
46          slave.SlaveState = SlaveState.Offline;
47        }
48      }
49    }
50
51    /// <summary>
52    /// Looks for parent tasks which have FinishWhenChildJobsFinished and set their state to finished
53    /// </summary>
54    private void FinishParentTasks(IPersistenceManager pm) {
55      var resourceDao = pm.ResourceDao;
56      var taskDao = pm.TaskDao;
57      var resourceIds = resourceDao.GetAll().Select(x => x.ResourceId).ToList();
58      var parentTasksToFinish = taskDao.GetParentTasks(resourceIds, 0, true);
59      foreach (var task in parentTasksToFinish) {
60        task.State = TaskState.Finished;
61        task.StateLogs.Add(new StateLog {
62          State = task.State,
63          SlaveId = null,
64          UserId = null,
65          Exception = string.Empty,
66          DateTime = DateTime.Now
67        });
68      }
69    }
70
71    /// <summary>
72    /// Looks for task which have not sent heartbeats for some time and reschedules them for calculation
73    /// </summary>
74    private void SetTimeoutTasksWaiting(IPersistenceManager pm) {
75      var taskDao = pm.TaskDao;
76      var tasks = taskDao.GetAll().Where(x => (x.State == TaskState.Calculating && (DateTime.Now - x.LastHeartbeat) > HeuristicLab.Services.Hive.Properties.Settings.Default.CalculatingJobHeartbeatTimeout)
77                                           || (x.State == TaskState.Transferring && (DateTime.Now - x.LastHeartbeat) > HeuristicLab.Services.Hive.Properties.Settings.Default.TransferringJobHeartbeatTimeout));
78      foreach (var task in tasks) {
79        task.State = TaskState.Waiting;
80        task.StateLogs.Add(new StateLog {
81          State = task.State,
82          SlaveId = null,
83          UserId = null,
84          Exception = SlaveTimeout,
85          DateTime = DateTime.Now
86        });
87        task.Command = null;
88      }
89    }
90
91    /// <summary>
92    /// Searches for slaves that are disposable and deletes them if they were offline for too long
93    /// </summary>
94    private void DeleteObsoleteSlaves(IPersistenceManager pm) {
95      var slaveDao = pm.SlaveDao;
96      var downtimeDao = pm.DowntimeDao;
97      var slaveIds = slaveDao.GetAll()
98        .Where(x => x.IsDisposable.GetValueOrDefault()
99                 && x.SlaveState == SlaveState.Offline
100                 && (DateTime.Now - x.LastHeartbeat) > Hive.Properties.Settings.Default.SweepInterval)
101        .Select(x => x.ResourceId)
102        .ToList();
103      foreach (var id in slaveIds) {
104        bool downtimesAvailable = downtimeDao.GetByResourceId(id).Any();
105        if (!downtimesAvailable) {
106          slaveDao.Delete(id);
107        }
108      }
109    }
110  }
111}
Note: See TracBrowser for help on using the repository browser.