Changeset 15641 for branches/HiveProjectManagement/HeuristicLab.Services.Hive

Timestamp:

01/23/18 13:51:46 (6 years ago)

Author:

jzenisek

Message:

#2839 finished implementation of job deletion routine

Location:

branches/HiveProjectManagement/HeuristicLab.Services.Hive/3.3

Files:

• HiveStatisticsGenerator.cs (modified) (5 diffs)
• Manager/HeartbeatManager.cs (modified) (1 diff)

branches/HiveProjectManagement/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

@@ -72 +72 @@
           });
         }
+
+        pm.UseTransaction(() => {
+          FlagJobsForDeletion(pm);
+          pm.SubmitChanges();
+        });
       }
     }
@@ -162 +167 @@
         dimJob.TotalTasks = totalTasks;
         dimJob.CompletedTasks = completedTasks;
+      }
+    }
+
+    private void FlagJobsForDeletion(PersistenceManager pm) {
+      var jobDao = pm.JobDao;
+      var jobs = jobDao.GetJobsReadyForDeletion();
+      foreach(var job in jobs) {
+        job.State = JobState.DeletionPending;
       }
     }
@@ -267 +280 @@
 
       var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
-      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
-        x.TaskId,
-        x.LastClientId
-      });
-
-      var newTasks =
+      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks();
+      //var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
+      //  x.TaskId,
+      //  x.LastClientId
+      //});
+
+      // query several properties for all new and not finished tasks
+      // in order to use them later eitheir
+      // (1) to update the fact task entry of not finished tasks
+      // (2) to insert a new fact task entry for new tasks
+      var newAndNotFinishedTasks =
         (from task in taskDao.GetAllChildTasks()
          let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
@@ -296 +314 @@
          }).ToList();
 
-      //insert facts for new tasks
+      // (1) update data of already existing facts
+      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = true
+      foreach (var notFinishedFactTask in notFinishedFactTasks) {
+        var nfftUpdate = newAndNotFinishedTasks.Where(x => x.TaskId == notFinishedFactTask.TaskId).SingleOrDefault();
+        if(nfftUpdate != null) {
+          var taskData = CalculateFactTaskData(nfftUpdate.StateLogs);
+
+          notFinishedFactTask.StartTime = taskData.StartTime;
+          notFinishedFactTask.EndTime = taskData.EndTime;
+          notFinishedFactTask.LastClientId = nfftUpdate.LastClientId;
+          notFinishedFactTask.Priority = nfftUpdate.Priority;
+          notFinishedFactTask.CoresRequired = nfftUpdate.CoresRequired;
+          notFinishedFactTask.MemoryRequired = nfftUpdate.MemoryRequired;
+          notFinishedFactTask.NumCalculationRuns = taskData.CalculationRuns;
+          notFinishedFactTask.NumRetries = taskData.Retries;
+          notFinishedFactTask.WaitingTime = taskData.WaitingTime;
+          notFinishedFactTask.CalculatingTime = taskData.CalculatingTime;
+          notFinishedFactTask.TransferTime = taskData.TransferTime;
+          notFinishedFactTask.TaskState = nfftUpdate.State;
+          notFinishedFactTask.Exception = taskData.Exception;
+          notFinishedFactTask.InitialWaitingTime = taskData.InitialWaitingTime;
+        }
+      }
+
+      // (2) insert facts for new tasks
+      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = false
       factTaskDao.Save(
-        from x in newTasks
+        from x in newAndNotFinishedTasks
         where !x.NotFinishedTask
         let taskData = CalculateFactTaskData(x.StateLogs)
@@ -320 +363 @@
         });
 
-      //update data of already existing facts
-      foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
-        var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
-        if (ntc.Any()) {
-          var x = ntc.Single();
-          var taskData = CalculateFactTaskData(x.StateLogs);
-
-          notFinishedTask.StartTime = taskData.StartTime;
-          notFinishedTask.EndTime = taskData.EndTime;
-          notFinishedTask.LastClientId = x.LastClientId;
-          notFinishedTask.Priority = x.Priority;
-          notFinishedTask.CoresRequired = x.CoresRequired;
-          notFinishedTask.MemoryRequired = x.MemoryRequired;
-          notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
-          notFinishedTask.NumRetries = taskData.Retries;
-          notFinishedTask.WaitingTime = taskData.WaitingTime;
-          notFinishedTask.CalculatingTime = taskData.CalculatingTime;
-          notFinishedTask.TransferTime = taskData.TransferTime;
-          notFinishedTask.TaskState = x.State;
-          notFinishedTask.Exception = taskData.Exception;
-          notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
-        }
-      }
+
+      ////update data of already existing facts
+      //foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
+      //  var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
+      //  if (ntc.Any()) {
+      //    var x = ntc.Single();
+      //    var taskData = CalculateFactTaskData(x.StateLogs);
+
+      //    notFinishedTask.StartTime = taskData.StartTime;
+      //    notFinishedTask.EndTime = taskData.EndTime;
+      //    notFinishedTask.LastClientId = x.LastClientId;
+      //    notFinishedTask.Priority = x.Priority;
+      //    notFinishedTask.CoresRequired = x.CoresRequired;
+      //    notFinishedTask.MemoryRequired = x.MemoryRequired;
+      //    notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
+      //    notFinishedTask.NumRetries = taskData.Retries;
+      //    notFinishedTask.WaitingTime = taskData.WaitingTime;
+      //    notFinishedTask.CalculatingTime = taskData.CalculatingTime;
+      //    notFinishedTask.TransferTime = taskData.TransferTime;
+      //    notFinishedTask.TaskState = x.State;
+      //    notFinishedTask.Exception = taskData.Exception;
+      //    notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
+      //  }
+      //}
     }
 

branches/HiveProjectManagement/HeuristicLab.Services.Hive/3.3/Manager/HeartbeatManager.cs

@@ -190 +190 @@
           // assigned resources ids of task do not match with slaveId (and parent resourceGroupIds); this might happen when slave is moved to different group
           actions.Add(new MessageContainer(MessageContainer.MessageType.PauseTask, curTask.TaskId));
-          LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not granted to calculate task: " + curTask.TaskId);
+          LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not granted to calculate task: " + curTask.TaskId + " of job: " + curTask.JobId);
         } else {
           // update task execution time