[6698] | 1 | #region License Information
|
---|
| 2 | /* HeuristicLab
|
---|
| 3 | * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
| 4 | *
|
---|
| 5 | * This file is part of HeuristicLab.
|
---|
| 6 | *
|
---|
| 7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
| 8 | * it under the terms of the GNU General Public License as published by
|
---|
| 9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
| 10 | * (at your option) any later version.
|
---|
| 11 | *
|
---|
| 12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
| 15 | * GNU General Public License for more details.
|
---|
| 16 | *
|
---|
| 17 | * You should have received a copy of the GNU General Public License
|
---|
| 18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
| 19 | */
|
---|
| 20 | #endregion
|
---|
| 21 |
|
---|
| 22 | using System;
|
---|
| 23 | using System.Collections.Generic;
|
---|
| 24 | using System.Linq;
|
---|
[6717] | 25 | using HeuristicLab.Services.Hive.DataAccess;
|
---|
| 26 | using DT = HeuristicLab.Services.Hive.DataTransfer;
|
---|
[6698] | 27 |
|
---|
[6717] | 28 |
|
---|
[6698] | 29 | namespace HeuristicLab.Services.Hive {
|
---|
| 30 | /// <summary>
|
---|
[6743] | 31 | /// This class offers methods for cleaning up offline slaves and task
|
---|
[6698] | 32 | /// </summary>
|
---|
| 33 | public class EventManager : IEventManager {
|
---|
[6717] | 34 | private IHiveDao dao {
|
---|
[6698] | 35 | get { return ServiceLocator.Instance.HiveDao; }
|
---|
| 36 | }
|
---|
| 37 | private IAuthorizationManager auth {
|
---|
| 38 | get { return ServiceLocator.Instance.AuthorizationManager; }
|
---|
| 39 | }
|
---|
| 40 | private ILogger log {
|
---|
| 41 | get { return LogFactory.GetLogger(this.GetType().Namespace); }
|
---|
| 42 | }
|
---|
| 43 |
|
---|
| 44 | public void Cleanup() {
|
---|
| 45 | log.Log("EventManager.Cleanup()");
|
---|
| 46 | SetTimeoutSlavesOffline();
|
---|
[6743] | 47 | SetTimeoutTasksWaiting();
|
---|
| 48 | FinishParentTasks();
|
---|
[6698] | 49 | UpdateStatistics();
|
---|
| 50 | }
|
---|
| 51 |
|
---|
| 52 | private void UpdateStatistics() {
|
---|
| 53 | var slaves = dao.GetSlaves(x => x.SlaveState == SlaveState.Calculating || x.SlaveState == SlaveState.Idle);
|
---|
| 54 |
|
---|
[6717] | 55 | var stats = new DataTransfer.Statistics();
|
---|
[6698] | 56 | stats.TimeStamp = DateTime.Now;
|
---|
[6717] | 57 | var slaveStats = new List<DT.SlaveStatistics>();
|
---|
[6698] | 58 | foreach (var slave in slaves) {
|
---|
[6717] | 59 | slaveStats.Add(new DT.SlaveStatistics() {
|
---|
[6698] | 60 | SlaveId = slave.Id,
|
---|
| 61 | Cores = slave.Cores.HasValue ? slave.Cores.Value : 0,
|
---|
| 62 | FreeCores = slave.FreeCores.HasValue ? slave.FreeCores.Value : 0,
|
---|
| 63 | Memory = slave.Memory.HasValue ? slave.Memory.Value : 0,
|
---|
| 64 | FreeMemory = slave.FreeMemory.HasValue ? slave.FreeMemory.Value : 0,
|
---|
| 65 | CpuUtilization = slave.CpuUtilization
|
---|
| 66 | });
|
---|
| 67 | }
|
---|
| 68 | stats.SlaveStatistics = slaveStats;
|
---|
[6946] | 69 | //collecting user statistics slows down the db and results in timeouts.
|
---|
| 70 | //we have to find another way to deal with this.
|
---|
| 71 | //until then the next line is commented out...
|
---|
| 72 | //stats.UserStatistics = dao.GetUserStatistics();
|
---|
[6698] | 73 | dao.AddStatistics(stats);
|
---|
| 74 | }
|
---|
| 75 |
|
---|
| 76 | /// <summary>
|
---|
[6743] | 77 | /// Searches for slaves which are timed out, puts them and their task offline
|
---|
[6698] | 78 | /// </summary>
|
---|
| 79 | private void SetTimeoutSlavesOffline() {
|
---|
| 80 | var slaves = dao.GetSlaves(x => x.SlaveState != SlaveState.Offline);
|
---|
[6717] | 81 | foreach (DT.Slave slave in slaves) {
|
---|
| 82 | if (!slave.LastHeartbeat.HasValue || (DateTime.Now - slave.LastHeartbeat.Value) > HeuristicLab.Services.Hive.Properties.Settings.Default.SlaveHeartbeatTimeout) {
|
---|
| 83 | slave.SlaveState = DT.SlaveState.Offline;
|
---|
[6743] | 84 | SetTasksWaiting(slave.Id);
|
---|
[6698] | 85 | dao.UpdateSlave(slave);
|
---|
| 86 | }
|
---|
| 87 | }
|
---|
| 88 | }
|
---|
| 89 |
|
---|
| 90 | /// <summary>
|
---|
[6743] | 91 | /// Looks for parent tasks which have FinishWhenChildJobsFinished and set their state to finished
|
---|
[6698] | 92 | /// </summary>
|
---|
[6743] | 93 | private void FinishParentTasks() {
|
---|
| 94 | var parentTasksToFinish = dao.GetParentTasks(dao.GetResources(x => true).Select(x => x.Id), 0, true);
|
---|
| 95 | foreach (var task in parentTasksToFinish) {
|
---|
| 96 | dao.UpdateTaskState(task.Id, TaskState.Finished, null, null, string.Empty);
|
---|
[6698] | 97 | }
|
---|
| 98 | }
|
---|
| 99 |
|
---|
[6743] | 100 | private void SetTasksWaiting(Guid slaveId) {
|
---|
| 101 | var tasks = dao.GetTasks(x => x.State == TaskState.Calculating).Where(x => x.StateLog.Last().SlaveId == slaveId);
|
---|
| 102 | foreach (var j in tasks) {
|
---|
| 103 | DT.Task task = dao.UpdateTaskState(j.Id, TaskState.Waiting, slaveId, null, "Slave timed out.");
|
---|
| 104 | task.Command = null;
|
---|
| 105 | dao.UpdateTask(task);
|
---|
[6698] | 106 | }
|
---|
| 107 | }
|
---|
| 108 |
|
---|
| 109 | /// <summary>
|
---|
[6743] | 110 | /// Looks for task which have not sent heartbeats for some time and reschedules them for calculation
|
---|
[6698] | 111 | /// </summary>
|
---|
[6743] | 112 | private void SetTimeoutTasksWaiting() {
|
---|
| 113 | var tasks = dao.GetTasks(x => (x.State == TaskState.Calculating && (DateTime.Now - x.LastHeartbeat) > HeuristicLab.Services.Hive.Properties.Settings.Default.CalculatingJobHeartbeatTimeout)
|
---|
[6721] | 114 | || (x.State == TaskState.Transferring && (DateTime.Now - x.LastHeartbeat) > HeuristicLab.Services.Hive.Properties.Settings.Default.TransferringJobHeartbeatTimeout));
|
---|
[6743] | 115 | foreach (var j in tasks) {
|
---|
| 116 | DT.Task task = dao.UpdateTaskState(j.Id, TaskState.Waiting, null, null, "Slave timed out.");
|
---|
| 117 | task.Command = null;
|
---|
| 118 | dao.UpdateTask(task);
|
---|
[6698] | 119 | }
|
---|
| 120 | }
|
---|
| 121 | }
|
---|
| 122 | }
|
---|