Changeset 17574 for trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/TaskDao.cs

Timestamp:

05/29/20 13:28:25 (4 years ago)

Author:

jkarder

Message:

#3062: overhauled statistics generation and cleanup

• switched to a single thread for database cleanup and statistics generation (executed sequentially)
• switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
• added code that aborts tasks whose jobs have already been marked for deletion
• added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
• updated DAO methods and introduced more bare metal sql
• introduced DAO methods for batch deletion
• fixed usage of enum values in DAO sql queries
• deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
• changed scheduling for less interference with janitor and other heartbeats
  • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
  • changed signature of ITaskScheduler.Schedule
  • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
  • changed RoundRobinTaskScheduler to use bare metal sql
• made MessageContainer a storable type (leftover)
• updated HiveJanitorServiceInstaller.nsi

File:

• trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/TaskDao.cs (modified) (4 diffs)

trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/TaskDao.cs

@@ -50 +50 @@
       //Because there is at the moment no case where this makes sense (there don't exist parent tasks which need to be calculated), 
       //we skip this step because it's wasted runtime
-      return DataContext.ExecuteQuery<TaskPriorityInfo>(GetWaitingTasksQueryString, 
-        slave.ResourceId, 
-        Enum.GetName(typeof(TaskState), TaskState.Waiting), 
-        slave.FreeCores, 
+      return DataContext.ExecuteQuery<TaskPriorityInfo>(GetWaitingTasksQueryString,
+        slave.ResourceId,
+        Enum.GetName(typeof(TaskState), TaskState.Waiting),
+        slave.FreeCores,
         slave.FreeMemory).ToList();
     }
@@ -65 +65 @@
     /// <returns></returns>
     public IEnumerable<Task> GetParentTasks(IEnumerable<Guid> resourceIds, int count, bool finished) {
-    var query = from t in Table
-                where t.State == TaskState.Waiting
-                    && t.IsParentTask
-                    && t.Job.AssignedJobResources.All(x => resourceIds.ToList().Contains(x.ResourceId))
-                    && t.FinishWhenChildJobsFinished == finished
-                    && t.ChildJobs.Any()
-                    && t.ChildJobs.All(x =>
-                      x.State == TaskState.Finished
-                      || x.State == TaskState.Aborted
-                      || x.State == TaskState.Failed)
+      var query = from t in Table
+                  where t.State == TaskState.Waiting
+                      && t.IsParentTask
+                      && t.Job.AssignedJobResources.All(x => resourceIds.ToList().Contains(x.ResourceId))
+                      && t.FinishWhenChildJobsFinished == finished
+                      && t.ChildJobs.Any()
+                      && t.ChildJobs.All(x =>
+                        x.State == TaskState.Finished
+                        || x.State == TaskState.Aborted
+                        || x.State == TaskState.Failed)
                   orderby t.Priority descending
                   select t;
@@ -82 +82 @@
     public void UpdateExecutionTime(Guid taskId, double executionTime) {
       DataContext.ExecuteCommand(UpdateExecutionTimeQuery, executionTime, DateTime.Now, taskId);
+    }
+
+    public int DeleteObsolete(int batchSize) {
+      return DataContext.ExecuteCommand(DeleteObsoleteQueryString, batchSize);
     }
 
@@ -129 +133 @@
        WHERE TaskId = {2}
     ";
+
+    private const string DeleteObsoleteQueryString = @"
+delete top ({0}) t1
+from task t1
+  left join task t2 on t1.taskid = t2.parenttaskid
+  join job j on j.jobid = t1.jobid
+where j.jobstate = 'deletionpending' and t2.taskid is null
+";
     #endregion
   }