#region License Information
/* HeuristicLab
* Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
*
* This file is part of HeuristicLab.
*
* HeuristicLab is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* HeuristicLab is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HeuristicLab. If not, see .
*/
#endregion
using System;
using System.Linq;
using HeuristicLab.Services.Hive.DataAccess;
using HeuristicLab.Services.Hive.DataAccess.Interfaces;
namespace HeuristicLab.Services.Hive.Manager {
public class EventManager : IEventManager {
private const string SlaveTimeout = "Slave timed out.";
private IPersistenceManager PersistenceManager {
get { return ServiceLocator.Instance.PersistenceManager; }
}
public void Cleanup() {
var pm = PersistenceManager;
pm.UseTransaction(() => {
FinishJobDeletion(pm);
pm.SubmitChanges();
});
pm.UseTransaction(() => {
SetTimeoutSlavesOffline(pm);
SetTimeoutTasksWaiting(pm);
DeleteObsoleteSlaves(pm);
pm.SubmitChanges();
});
pm.UseTransaction(() => {
FinishParentTasks(pm);
pm.SubmitChanges();
});
}
///
/// Deletes all jobs which are in state "DeletionPending" (this will include all corresponding tasks).
/// The state "DeletionPending" is set by HiveJanitor > StatisticsGenerator
///
private void FinishJobDeletion(IPersistenceManager pm) {
var jobDao = pm.JobDao;
jobDao.DeleteByState(JobState.DeletionPending);
}
///
/// Searches for slaves which are timed out, puts them and their task offline
///
private void SetTimeoutSlavesOffline(IPersistenceManager pm) {
var slaveDao = pm.SlaveDao;
var slaves = slaveDao.GetOnlineSlaves();
foreach (var slave in slaves) {
if (!slave.LastHeartbeat.HasValue ||
(DateTime.Now - slave.LastHeartbeat.Value) >
Properties.Settings.Default.SlaveHeartbeatTimeout) {
slave.SlaveState = SlaveState.Offline;
}
}
}
///
/// Looks for parent tasks which have FinishWhenChildJobsFinished and set their state to finished
///
private void FinishParentTasks(IPersistenceManager pm) {
var resourceDao = pm.ResourceDao;
var taskDao = pm.TaskDao;
var resourceIds = resourceDao.GetAll().Select(x => x.ResourceId).ToList();
var parentTasksToFinish = taskDao.GetParentTasks(resourceIds, 0, true);
foreach (var task in parentTasksToFinish) {
task.State = TaskState.Finished;
task.StateLogs.Add(new StateLog {
State = task.State,
SlaveId = null,
UserId = null,
Exception = string.Empty,
DateTime = DateTime.Now
});
}
}
///
/// Looks for task which have not sent heartbeats for some time and reschedules them for calculation
///
private void SetTimeoutTasksWaiting(IPersistenceManager pm) {
var taskDao = pm.TaskDao;
var tasks = taskDao.GetAll().Where(x => (x.State == TaskState.Calculating && (DateTime.Now - x.LastHeartbeat) > Properties.Settings.Default.CalculatingJobHeartbeatTimeout)
|| (x.State == TaskState.Transferring && (DateTime.Now - x.LastHeartbeat) > Properties.Settings.Default.TransferringJobHeartbeatTimeout));
foreach (var task in tasks) {
task.State = TaskState.Waiting;
task.StateLogs.Add(new StateLog {
State = task.State,
SlaveId = null,
UserId = null,
Exception = SlaveTimeout,
DateTime = DateTime.Now
});
task.Command = null;
}
}
///
/// Searches for slaves that are disposable and deletes them if they were offline for too long
///
private void DeleteObsoleteSlaves(IPersistenceManager pm) {
var slaveDao = pm.SlaveDao;
var downtimeDao = pm.DowntimeDao;
var slaveIds = slaveDao.GetAll()
.Where(x => x.IsDisposable.GetValueOrDefault()
&& x.SlaveState == SlaveState.Offline
&& (DateTime.Now - x.LastHeartbeat) > Properties.Settings.Default.SweepInterval)
.Select(x => x.ResourceId)
.ToList();
foreach (var id in slaveIds) {
bool downtimesAvailable = downtimeDao.GetByResourceId(id).Any();
if (!downtimesAvailable) {
slaveDao.Delete(id);
}
}
}
}
}