Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/LifecycleManager.cs @ 5786

Last change on this file since 5786 was 5786, checked in by cneumuel, 13 years ago

#1233

  • implemented correct numbering of BatchRuns
  • improvements in ExperimentManager
  • fixed bug in server (jobs were scheduled multiple times)
  • added exception handling for task in slave
  • improved timeout handling of jobs (LifecycleManager)
File size: 3.0 KB
Line 
1using System;
2using System.Linq;
3using HeuristicLab.Services.Hive.Common;
4using HeuristicLab.Services.Hive.Common.DataTransfer;
5
6namespace HeuristicLab.Services.Hive {
7  /// <summary>
8  /// This class offers methods for cleaning up offline slaves and jobs
9  /// </summary>
10  public class LifecycleManager : ILifecycleManager {
11    private DataAccess.IHiveDao dao {
12      get { return ServiceLocator.Instance.HiveDao; }
13    }
14    private HeuristicLab.Services.Hive.DataAccess.TransactionManager trans {
15      get { return ServiceLocator.Instance.TransactionManager; }
16    }
17    private IAuthorizationManager auth {
18      get { return ServiceLocator.Instance.AuthorizationManager; }
19    }
20    private ILogger log {
21      get { return LogFactory.GetLogger(this.GetType().Namespace); }
22    }
23
24    public void Cleanup() {
25      log.Log("LifecycleManager.Cleanup()");
26      SetTimeoutSlavesOffline();
27      SetTimeoutJobsWaiting();
28      FinishParentJobs();     
29    }
30
31    /// <summary>
32    /// Searches for slaves which are timed out, puts them and their jobs offline
33    /// </summary>
34    private void SetTimeoutSlavesOffline() {
35      var slaves = dao.GetSlaves(x => x.SlaveState != SlaveState.Offline);
36      foreach (Slave slave in slaves) {
37        if (!slave.LastHeartbeat.HasValue || (DateTime.Now - slave.LastHeartbeat.Value) > ApplicationConstants.SlaveHeartbeatTimeout) {
38          slave.SlaveState = SlaveState.Offline;
39          SetJobsWaiting(slave.Id);
40          dao.UpdateSlave(slave);
41        }
42      }
43    }
44
45    /// <summary>
46    /// Looks for parent jobs which have FinishWhenChildJobsFinished and set their state to finished
47    /// </summary>
48    private void FinishParentJobs() {
49      var parentJobsToFinish = dao.GetParentJobs(dao.GetResources(x => true).Select(x => x.Id), 0, true);
50      foreach (var job in parentJobsToFinish) {
51        dao.UpdateJobState(job.Id, JobState.Finished, null, null, string.Empty);
52      }
53    }
54
55    private void SetJobsWaiting(Guid slaveId) {
56      var jobs = dao.GetJobs(x => x.State == JobState.Calculating).Where(x => x.StateLog.Last().SlaveId == slaveId);
57      foreach (var j in jobs) {
58        Job job = dao.UpdateJobState(j.Id, JobState.Waiting, slaveId, null, "Slave timed out.");
59        job.Command = null;
60        dao.UpdateJob(job);
61      }
62    }
63
64    /// <summary>
65    /// Looks for jobs which have not sent heartbeats for some time and reschedules them for calculation
66    /// </summary>
67    private void SetTimeoutJobsWaiting() {
68      var jobs = dao.GetJobs(x => (x.State == JobState.Calculating && (DateTime.Now - x.LastHeartbeat) > ApplicationConstants.CalculatingJobHeartbeatTimeout)
69                               || (x.State == JobState.Transferring && (DateTime.Now - x.LastHeartbeat) > ApplicationConstants.TransferringJobHeartbeatTimeout));
70      foreach (var j in jobs) {
71        Job job = dao.UpdateJobState(j.Id, JobState.Waiting, null, null, "Slave timed out.");
72        job.Command = null;
73        dao.UpdateJob(job);
74      }
75    }
76  }
77}
Note: See TracBrowser for help on using the repository browser.