Changeset 16141 for branches/2817-BinPackingSpeedup/HeuristicLab.Services.Hive/3.3/Manager

Timestamp:

09/14/18 11:47:37 (6 years ago)

Author:

abeham

Message:

#2817: updated to trunk r16140

Location:

branches/2817-BinPackingSpeedup

Files:

• . (modified) (1 prop)
• HeuristicLab.Services.Hive (modified) (1 prop)
• HeuristicLab.Services.Hive/3.3/Manager/AuthorizationManager.cs (modified) (5 diffs)
• HeuristicLab.Services.Hive/3.3/Manager/EventManager.cs (modified) (2 diffs)
• HeuristicLab.Services.Hive/3.3/Manager/HeartbeatManager.cs (modified) (1 diff)

branches/2817-BinPackingSpeedup/HeuristicLab.Services.Hive/3.3/Manager/AuthorizationManager.cs

@@ -27 +27 @@
 using DA = HeuristicLab.Services.Hive.DataAccess;
 using DT = HeuristicLab.Services.Hive.DataTransfer;
-
+using System.Collections.Generic;
+using System.Linq;
 
 namespace HeuristicLab.Services.Hive {
   public class AuthorizationManager : IAuthorizationManager {
 
-    private const string NOT_AUTHORIZED = "Current user is not authorized to access the requested resource";
+    private const string NOT_AUTHORIZED_USERRESOURCE = "Current user is not authorized to access the requested resource";
+    private const string NOT_AUTHORIZED_USERPROJECT = "Current user is not authorized to access the requested project";
+    private const string NOT_AUTHORIZED_USERJOB = "Current user is not authorized to access the requested job";
+    private const string NOT_AUTHORIZED_PROJECTRESOURCE = "Selected project is not authorized to access the requested resource";
+    private const string USER_NOT_IDENTIFIED = "User could not be identified";
+    private const string TASK_NOT_EXISTENT = "Queried task could not be found";
+    private const string PROJECT_NOT_EXISTENT = "Queried project could not be found";
+
     private IPersistenceManager PersistenceManager {
       get { return ServiceLocator.Instance.PersistenceManager; }
@@ -47 +55 @@
     public void Authorize(Guid userId) {
       if (userId != ServiceLocator.Instance.UserManager.CurrentUserId)
-        throw new SecurityException(NOT_AUTHORIZED);
+        throw new SecurityException(NOT_AUTHORIZED_USERRESOURCE);
     }
 
@@ -56 +64 @@
       pm.UseTransaction(() => {
         var task = taskDao.GetById(taskId);
-        if (task == null) throw new SecurityException(NOT_AUTHORIZED);
+        if (task == null) throw new SecurityException(TASK_NOT_EXISTENT);
         AuthorizeJob(pm, task.JobId, requiredPermission);
       });
@@ -68 +76 @@
     }
 
+    // authorize if user is admin or resource owner
     public void AuthorizeForResourceAdministration(Guid resourceId) {
+      var currentUserId = UserManager.CurrentUserId;
       var pm = PersistenceManager;
       var resourceDao = pm.ResourceDao;
       pm.UseTransaction(() => {
         var resource = resourceDao.GetById(resourceId);
-        if (resource == null) throw new SecurityException(NOT_AUTHORIZED);
-        if (resource.OwnerUserId != UserManager.CurrentUserId
+        if (resource == null) throw new SecurityException(NOT_AUTHORIZED_USERRESOURCE);
+
+        if (resource.OwnerUserId != currentUserId
             && !RoleVerifier.IsInRole(HiveRoles.Administrator)) {
-          throw new SecurityException(NOT_AUTHORIZED);
-        }
-      });
+          throw new SecurityException(NOT_AUTHORIZED_USERRESOURCE);
+        }
+      });
+    }
+
+    // authorize if user is admin, project owner or owner of a parent project
+    public void AuthorizeForProjectAdministration(Guid projectId, bool parentalOwnership) {
+      if (projectId == null || projectId == Guid.Empty) return;
+      var currentUserId = UserManager.CurrentUserId;
+      var pm = PersistenceManager;
+      var projectDao = pm.ProjectDao;
+      pm.UseTransaction(() => {
+        var project = projectDao.GetById(projectId);
+        if (project == null) throw new ArgumentException(PROJECT_NOT_EXISTENT);
+        if(!RoleVerifier.IsInRole(HiveRoles.Administrator)
+          && !project.ParentProjectId.HasValue) {
+          throw new SecurityException(NOT_AUTHORIZED_USERPROJECT);
+        }
+
+        List<Project> projectBranch = null;
+        if(parentalOwnership) projectBranch = projectDao.GetParentProjectsById(projectId).ToList();
+        else projectBranch = projectDao.GetCurrentAndParentProjectsById(projectId).ToList();
+
+        if(!RoleVerifier.IsInRole(HiveRoles.Administrator)
+            && !projectBranch.Select(x => x.OwnerUserId).Contains(currentUserId)) {
+          throw new SecurityException(NOT_AUTHORIZED_USERPROJECT);
+        }
+      });
+    }
+
+    // authorize if user is admin, or owner of a project or parent project, for which the resources are assigned to
+    public void AuthorizeForProjectResourceAdministration(Guid projectId, IEnumerable<Guid> resourceIds) {
+      if (projectId == null || projectId == Guid.Empty) return;
+      var currentUserId = UserManager.CurrentUserId;
+      var pm = PersistenceManager;
+      var projectDao = pm.ProjectDao;
+      var resourceDao = pm.ResourceDao;
+      var assignedProjectResourceDao = pm.AssignedProjectResourceDao;
+      pm.UseTransaction(() => {
+        // check if project exists (not necessary)
+        var project = projectDao.GetById(projectId);
+        if (project == null) throw new SecurityException(NOT_AUTHORIZED_USERRESOURCE);
+
+        // check if resourceIds exist
+        if (resourceIds != null && resourceIds.Any() && !resourceDao.CheckExistence(resourceIds))
+          throw new SecurityException(NOT_AUTHORIZED_USERRESOURCE);
+
+        // check if user is admin
+        if (RoleVerifier.IsInRole(HiveRoles.Administrator)) return;
+
+        // check if user is owner of the project or a parent project
+        var projectBranch = projectDao.GetCurrentAndParentProjectsById(projectId).ToList();
+        if (!projectBranch.Select(x => x.OwnerUserId).Contains(currentUserId)
+            && !RoleVerifier.IsInRole(HiveRoles.Administrator)) {
+          throw new SecurityException(NOT_AUTHORIZED_USERPROJECT);
+        }
+
+        // check if the all argument resourceIds are among the assigned resources of the owned projects
+        var grantedResourceIds = assignedProjectResourceDao.GetAllGrantedResourceIdsOfOwnedParentProjects(projectId, currentUserId).ToList();
+        if(resourceIds.Except(grantedResourceIds).Any()) {
+          throw new SecurityException(NOT_AUTHORIZED_USERRESOURCE);
+        }
+      });
+    }
+
+    // Check if a project is authorized to use a list of resources
+    public void AuthorizeProjectForResourcesUse(Guid projectId, IEnumerable<Guid> resourceIds) {
+      if (projectId == null || projectId == Guid.Empty || resourceIds == null || !resourceIds.Any()) return;
+      var pm = PersistenceManager;
+      var assignedProjectResourceDao = pm.AssignedProjectResourceDao;
+      if (!assignedProjectResourceDao.CheckProjectGrantedForResources(projectId, resourceIds))
+        throw new SecurityException(NOT_AUTHORIZED_PROJECTRESOURCE);
+    }
+
+    // Check if current user is authorized to use an explicit project (e.g. in order to add a job)
+    // note: administrators and project owner are NOT automatically granted
+    public void AuthorizeUserForProjectUse(Guid userId, Guid projectId) {
+      if(userId == null || userId == Guid.Empty) {
+        throw new SecurityException(USER_NOT_IDENTIFIED);
+      }
+      if(projectId == null) return;
+
+      var pm = PersistenceManager;
+      // collect current and group membership Ids
+      var userAndGroupIds = new List<Guid>() { userId };
+      userAndGroupIds.AddRange(UserManager.GetUserGroupIdsOfUser(userId));
+      // perform the actual check
+      var projectPermissionDao = pm.ProjectPermissionDao;
+      if (!projectPermissionDao.CheckUserGrantedForProject(projectId, userAndGroupIds)) {
+        throw new SecurityException(NOT_AUTHORIZED_USERPROJECT);
+      }
     }
 
@@ -93 +192 @@
 
     private void AuthorizeJob(IPersistenceManager pm, Guid jobId, DT.Permission requiredPermission) {
+      var currentUserId = UserManager.CurrentUserId;
       var requiredPermissionEntity = requiredPermission.ToEntity();
-      DA.Permission permission = GetPermissionForJob(pm, jobId, UserManager.CurrentUserId);
+      DA.Permission permission = GetPermissionForJob(pm, jobId, currentUserId);
       if (permission == Permission.NotAllowed
           || ((permission != requiredPermissionEntity) && requiredPermissionEntity == Permission.Full)) {
-        throw new SecurityException(NOT_AUTHORIZED);
+        throw new SecurityException(NOT_AUTHORIZED_USERJOB);
       }
     }

branches/2817-BinPackingSpeedup/HeuristicLab.Services.Hive/3.3/Manager/EventManager.cs

@@ -34 +34 @@
     public void Cleanup() {
       var pm = PersistenceManager;
+
+      pm.UseTransaction(() => {
+        FinishJobDeletion(pm);
+        pm.SubmitChanges();
+      });
+
       pm.UseTransaction(() => {
         SetTimeoutSlavesOffline(pm);
@@ -45 +51 @@
         pm.SubmitChanges();
       });
+    }
+
+    /// <summary>
+    /// Deletes all jobs which are in state "DeletionPending" (this will include all corresponding tasks).
+    /// The state "DeletionPending" is set by HiveJanitor > StatisticsGenerator
+    /// </summary>
+    private void FinishJobDeletion(IPersistenceManager pm) {
+      var jobDao = pm.JobDao;
+      jobDao.DeleteByState(JobState.DeletionPending);
     }
 

branches/2817-BinPackingSpeedup/HeuristicLab.Services.Hive/3.3/Manager/HeartbeatManager.cs

@@ -142 +142 @@
     private IEnumerable<MessageContainer> UpdateTasks(IPersistenceManager pm, Heartbeat heartbeat, bool isAllowedToCalculate) {
       var taskDao = pm.TaskDao;
-      var assignedResourceDao = pm.AssignedResourceDao;
+      var jobDao = pm.JobDao;
+      var assignedJobResourceDao = pm.AssignedJobResourceDao;
       var actions = new List<MessageContainer>();
       if (heartbeat.JobProgress == null || !heartbeat.JobProgress.Any())
         return actions;
 
-      if (!isAllowedToCalculate && heartbeat.JobProgress.Count != 0) {
-        actions.Add(new MessageContainer(MessageContainer.MessageType.PauseAll));
-      } else {
-        // select all tasks and statelogs with one query
-        var taskIds = heartbeat.JobProgress.Select(x => x.Key).ToList();
-        var taskInfos = pm.UseTransaction(() =>
-          (from task in taskDao.GetAll()
-           where taskIds.Contains(task.TaskId)
-           let lastStateLog = task.StateLogs.OrderByDescending(x => x.DateTime).FirstOrDefault()
-           select new {
-             TaskId = task.TaskId,
-             Command = task.Command,
-             SlaveId = lastStateLog != null ? lastStateLog.SlaveId : default(Guid)
-           }).ToList()
-        );
-
-        // process the jobProgresses
-        foreach (var jobProgress in heartbeat.JobProgress) {
-          var progress = jobProgress;
-          var curTask = taskInfos.SingleOrDefault(x => x.TaskId == progress.Key);
-          if (curTask == null) {
-            actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, progress.Key));
-            LogFactory.GetLogger(this.GetType().Namespace).Log("Task on slave " + heartbeat.SlaveId + " does not exist in DB: " + jobProgress.Key);
-          } else {
-            var slaveId = curTask.SlaveId;
-            if (slaveId == Guid.Empty || slaveId != heartbeat.SlaveId) {
-              // assigned slave does not match heartbeat
+      var jobIdsWithStatisticsPending = jobDao.GetJobIdsByState(DA.JobState.StatisticsPending).ToList();
+
+      // select all tasks and statelogs with one query
+      var taskIds = heartbeat.JobProgress.Select(x => x.Key).ToList();
+      var taskInfos = pm.UseTransaction(() =>
+        (from task in taskDao.GetAll()
+          where taskIds.Contains(task.TaskId)
+          let lastStateLog = task.StateLogs.OrderByDescending(x => x.DateTime).FirstOrDefault()
+          select new {
+            TaskId = task.TaskId,
+            JobId = task.JobId,
+            State = task.State,
+            Command = task.Command,
+            SlaveId = lastStateLog != null ? lastStateLog.SlaveId : default(Guid)
+          }).ToList()
+      );
+
+      // process the jobProgresses
+      foreach (var jobProgress in heartbeat.JobProgress) {
+        var progress = jobProgress;
+        var curTask = taskInfos.SingleOrDefault(x => x.TaskId == progress.Key);
+        if (curTask == null) {
+          actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, progress.Key));
+          LogFactory.GetLogger(this.GetType().Namespace).Log("Task on slave " + heartbeat.SlaveId + " does not exist in DB: " + jobProgress.Key);
+        } else if (jobIdsWithStatisticsPending.Contains(curTask.JobId)) {
+          // parenting job of current task has been requested for deletion (indicated by job state "Statistics Pending")
+          // update task execution time
+          pm.UseTransaction(() => {
+            taskDao.UpdateExecutionTime(curTask.TaskId, progress.Value.TotalMilliseconds);
+          });
+          actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
+          LogFactory.GetLogger(this.GetType().Namespace).Log("Abort task " + curTask.TaskId + " on slave " + heartbeat.SlaveId + ". The parenting job " + curTask.JobId + " was requested to be deleted.");
+        } else if (curTask.SlaveId == Guid.Empty || curTask.SlaveId != heartbeat.SlaveId) {
+          // assigned slave does not match heartbeat
+          actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
+          LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not supposed to calculate task: " + curTask.TaskId);
+        } else if (!isAllowedToCalculate) {
+          actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
+          LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not allowed to calculate any tasks tue to a downtime. The task is paused.");
+        } else if (!assignedJobResourceDao.CheckJobGrantedForResource(curTask.JobId, heartbeat.SlaveId)) {
+          // slaveId (and parent resourceGroupIds) are not among the assigned resources ids for task-parenting job
+          // this might happen when (a) job-resource assignment has been changed (b) slave is moved to different group
+          actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
+          LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not granted to calculate task: " + curTask.TaskId + " of job: " + curTask.JobId);
+        } else {
+          // update task execution time
+          pm.UseTransaction(() => {
+            taskDao.UpdateExecutionTime(curTask.TaskId, progress.Value.TotalMilliseconds);
+          });
+          switch (curTask.Command) {
+            case DA.Command.Stop:
+              actions.Add(new MessageContainer(MessageContainer.MessageType.StopTask, curTask.TaskId));
+              break;
+            case DA.Command.Pause:
+              actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
+              break;
+            case DA.Command.Abort:
               actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
-              LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not supposed to calculate task: " + curTask.TaskId);
-            } else if (!assignedResourceDao.TaskIsAllowedToBeCalculatedBySlave(curTask.TaskId, heartbeat.SlaveId)) {
-              // assigned resources ids of task do not match with slaveId (and parent resourceGroupIds); this might happen when slave is moved to different group
-              actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
-            } else {
-              // update task execution time
-              pm.UseTransaction(() => {
-                taskDao.UpdateExecutionTime(curTask.TaskId, progress.Value.TotalMilliseconds);
-              });
-              switch (curTask.Command) {
-                case DA.Command.Stop:
-                  actions.Add(new MessageContainer(MessageContainer.MessageType.StopTask, curTask.TaskId));
-                  break;
-                case DA.Command.Pause:
-                  actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
-                  break;
-                case DA.Command.Abort:
-                  actions.Add(new MessageContainer(MessageContainer.MessageType.AbortTask, curTask.TaskId));
-                  break;
-              }
-            }
-          }
-        }
-      }
+              break;
+          }
+        }
+        
+      } 
       return actions;
     }