source: branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/HeartbeatManager.cs @ 5597

using System;
using System.Collections.Generic;
using System.Linq;
using HeuristicLab.Services.Hive.Common;
using HeuristicLab.Services.Hive.Common.DataTransfer;
using HeuristicLab.Tracing;

namespace HeuristicLab.Services.Hive {
  public class HeartbeatManager {
    private DataAccess.IHiveDao dao {
      get { return ServiceLocator.Instance.HiveDao; }
    }
    private HeuristicLab.Services.Hive.DataAccess.TransactionManager trans {
      get { return ServiceLocator.Instance.TransactionManager; }
    }
    private IAuthorizationManager auth {
      get { return ServiceLocator.Instance.AuthorizationManager; }
    }

    /// <summary>
    /// This method will be called every time a slave sends a heartbeat (-> very often; concurrency is important!)
    /// </summary>
    /// <returns>a list of actions the slave should do</returns>
    public List<MessageContainer> ProcessHeartbeat(Heartbeat heartbeat) {
      List<MessageContainer> actions = new List<MessageContainer>();
      Slave slave = dao.GetSlave(heartbeat.SlaveId);
      if (slave == null) {
        actions.Add(new MessageContainer(MessageContainer.MessageType.SayHello));
      } else {
        // update slave data
        slave.FreeCores = heartbeat.FreeCores;
        slave.FreeMemory = heartbeat.FreeMemory;
        slave.IsAllowedToCalculate = true; // Todo: look into calendar
        slave.SlaveState = (heartbeat.JobProgress != null && heartbeat.JobProgress.Count > 0) ? SlaveState.Calculating : SlaveState.Idle;
        slave.LastHeartbeat = DateTime.Now;
        dao.UpdateSlave(slave);

        // update job data
        actions.AddRange(UpdateJobs(heartbeat));

        // assign new job
        if (heartbeat.AssignJob && this.IsAllowedToSendJobs() && slave.IsAllowedToCalculate && heartbeat.FreeCores > 0) {
          var availableJobs = dao.GetWaitingJobs(slave, 1);
          if (availableJobs.Count() > 0) {
            var job = availableJobs.First();
            actions.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob, job.Id));
            AssignJob(slave, job);
          }
        }
      }
      return actions;
    }

    private void AssignJob(Slave slave, Job job) {
      job.SetState(JobState.Transferring, slave.Id, "");
      dao.UpdateJob(job);
      dao.UpdateSlave(slave);
    }

    /// <summary>
    /// Update the progress of each job
    /// Checks if all the jobs sent by heartbeat are supposed to be calculated by this slave
    /// </summary>
    private IEnumerable<MessageContainer> UpdateJobs(Heartbeat heartbeat) {
      List<MessageContainer> actions = new List<MessageContainer>();

      if (heartbeat.JobProgress == null)
        return actions;

      // process the jobProgresses
      foreach (var jobProgress in heartbeat.JobProgress) {
        Job curJob = dao.GetJob(jobProgress.Key);
        if (curJob == null) {
          // job does not exist in db
          actions.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, jobProgress.Key));
          Logger.Error("Job does not exist in DB: " + jobProgress.Key);
        } else {
          if (curJob.CurrentStateLog.SlaveId == Guid.Empty || curJob.CurrentStateLog.SlaveId != heartbeat.SlaveId) {
            // assigned slave does not match heartbeat
            actions.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, curJob.Id));
            Logger.Error("The slave " + heartbeat.SlaveId + " is not supposed to calculate Job: " + curJob);
          } else {
            // save job execution time
            curJob.ExecutionTime = jobProgress.Value;
            curJob.LastHeartbeat = DateTime.Now;

            if (curJob.State == JobState.Aborted) {
              // a request to abort the job has been set
              actions.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, curJob.Id));
            } else if (curJob.State != JobState.Calculating) {
              // jobstate was 'Transferring' before, now calculating
              curJob.SetState(JobState.Calculating, heartbeat.SlaveId, "");
            }
            dao.UpdateJob(curJob);
          }
        }
      }
      return actions;
    }

    /// <summary>
    /// Returns true if there are enough resources to send a job
    /// There should not be too many jobs sent simultaniously
    /// </summary>
    private bool IsAllowedToSendJobs() {
      return true; // JobsCurrentlyTransferring < ApplicationConstants.MaxJobTransferCount;
      // Todo: see if unlimited job transfer count works. if not, look into db and count jobs in state Transferring
    }
  }
}