Changeset 16240 for branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

Timestamp:

10/18/18 16:16:31 (6 years ago)

Author:

gkronber

Message:

#2915: merged changes in the trunk up to current HEAD (r15951:16232) into the branch

Location:

branches/2915-AbsoluteSymbol

Files:

• . (modified) (1 prop)
• HeuristicLab.Services.Hive (modified) (1 prop)
• HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs (modified) (16 diffs)

branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

@@ -37 +37 @@
     public void GenerateStatistics() {
       using (var pm = new PersistenceManager()) {
+
+        pm.UseTransaction(() => {
+          UpdateDimProjectTable(pm);
+          pm.SubmitChanges();
+        });
+
         pm.UseTransaction(() => {
           UpdateDimUserTable(pm);
+          
           UpdateDimJobTable(pm);
           UpdateDimClientsTable(pm);
@@ -54 +61 @@
             UpdateFactClientInfoTable(time, pm);
             pm.SubmitChanges();
+            UpdateFactProjectInfoTable(time, pm);
+            pm.SubmitChanges();
           });
 
           pm.UseTransaction(() => {
-            UpdateTaskFactsTable(pm);
+            UpdateFactTaskTable(pm);
             try {
               pm.SubmitChanges();
@@ -72 +81 @@
           });
         }
+
+        pm.UseTransaction(() => {
+          FlagJobsForDeletion(pm);
+          pm.SubmitChanges();
+        });
       }
     }
@@ -106 +120 @@
     }
 
+    // add new projects
+    // delete expired projects
+    // update information of existing projects
+    private void UpdateDimProjectTable(PersistenceManager pm) {
+      var projectDao = pm.ProjectDao;
+      var dimProjectDao = pm.DimProjectDao;
+
+      var projects = projectDao.GetAll().ToList();
+      var dimProjects = dimProjectDao.GetAllOnlineProjects().ToList();
+
+      var onlineProjects = dimProjects.Where(x => projects.Select(y => y.ProjectId).Contains(x.ProjectId));
+      var addedProjects = projects.Where(x => !dimProjects.Select(y => y.ProjectId).Contains(x.ProjectId));
+      var removedProjects = dimProjects.Where(x => !projects.Select(y => y.ProjectId).Contains(x.ProjectId));
+
+      // set expiration time of removed projects
+      foreach (var p in removedProjects) {
+        p.DateExpired = DateTime.Now;
+      }
+
+      // add new projects
+      dimProjectDao.Save(addedProjects.Select(x => new DimProject {
+        ProjectId = x.ProjectId,
+        ParentProjectId = x.ParentProjectId,
+        Name = x.Name,
+        Description = x.Description,
+        OwnerUserId = x.OwnerUserId,
+        StartDate = x.StartDate,
+        EndDate = x.EndDate,
+        DateCreated = x.DateCreated,
+        DateExpired = null
+      }));
+
+      // expire project if its parent has changed and create a new entry
+      // otherwise perform "normal" update
+      foreach (var dimP in onlineProjects) {
+        var p = projects.Where(x => x.ProjectId == dimP.ProjectId).SingleOrDefault();
+        if (p != null) {
+          if (dimP.ParentProjectId == null ? p.ParentProjectId != null : dimP.ParentProjectId != p.ParentProjectId) { // or: (!object.Equals(dimP.ParentProjectId, p.ParentProjectId))
+            dimP.DateExpired = DateTime.Now;
+            dimProjectDao.Save(new DimProject {
+              ProjectId = p.ProjectId,
+              ParentProjectId = p.ParentProjectId,
+              Name = p.Name,
+              Description = p.Description,
+              OwnerUserId = p.OwnerUserId,
+              StartDate = p.StartDate,
+              EndDate = p.EndDate,
+              DateCreated = p.DateCreated,
+              DateExpired = null
+            });
+          } else {
+            dimP.Name = p.Name;
+            dimP.Description = p.Description;
+            dimP.OwnerUserId = p.OwnerUserId;
+            dimP.StartDate = p.StartDate;
+            dimP.EndDate = p.EndDate;
+          }
+        }
+      }
+    }
+
     private void UpdateDimJobTable(PersistenceManager pm) {
+      var dimProjectDao = pm.DimProjectDao;
       var dimJobDao = pm.DimJobDao;
       var jobDao = pm.JobDao;
@@ -118 +194 @@
           JobName = x.Name ?? string.Empty,
           DateCreated = x.DateCreated,
+          ProjectId = dimProjectDao.GetLastValidIdByProjectId(x.ProjectId),
           TotalTasks = taskDao.GetAll().Count(y => y.JobId == x.JobId)
         })
@@ -127 +204 @@
         UserName = GetUserName(x.UserId),
         DateCreated = x.DateCreated,
+        ProjectId = x.ProjectId,
         TotalTasks = x.TotalTasks,
         CompletedTasks = 0,
@@ -134 +212 @@
 
     private void UpdateExistingDimJobs(PersistenceManager pm) {
+      var dimProjectDao = pm.DimProjectDao;
       var jobDao = pm.JobDao;
       var dimJobDao = pm.DimJobDao;
@@ -151 +230 @@
           }
         }
+        var job = jobDao.GetById(dimJob.JobId);
         if (totalTasks == completedTasks) {
           var completeDate = factTaskDao.GetLastCompletedTaskFromJob(dimJob.JobId);
           if (completeDate == null) {
-            if (jobDao.GetById(dimJob.JobId) == null) {
+            if (job == null) {
               completeDate = DateTime.Now;
             }
@@ -160 +240 @@
           dimJob.DateCompleted = completeDate;
         }
+        if(job != null) {
+          dimJob.JobName = job.Name;
+          dimJob.ProjectId = dimProjectDao.GetLastValidIdByProjectId(job.ProjectId);
+        }
+
         dimJob.TotalTasks = totalTasks;
         dimJob.CompletedTasks = completedTasks;
@@ -165 +250 @@
     }
 
+    private void FlagJobsForDeletion(PersistenceManager pm) {
+      var jobDao = pm.JobDao;
+      var jobs = jobDao.GetJobsReadyForDeletion();
+      foreach(var job in jobs) {
+        job.State = JobState.DeletionPending;
+      }
+    }
+
     private void UpdateDimClientsTable(PersistenceManager pm) {
       var dimClientDao = pm.DimClientDao;
-      var slaveDao = pm.SlaveDao;
-      var slaves = slaveDao.GetAll();
-      var recentlyAddedClients = dimClientDao.GetActiveClients();
-      var slaveIds = slaves.Select(x => x.ResourceId);
-
-      var removedClientIds = recentlyAddedClients
-        .Where(x => !slaveIds.Contains(x.ResourceId))
-        .Select(x => x.Id);
-      var modifiedClients =
-        from slave in slaves
-        join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
-        where (slave.Name != client.Name
-               || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
-               || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
-               || slave.ParentResourceId != client.ResourceGroupId
-               || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
-        select new {
-          SlaveId = slave.ResourceId,
-          ClientId = client.Id
-        };
-      var clientIds = dimClientDao.GetActiveClients().Select(x => x.ResourceId);
-      var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
-      var newClients = slaves
-        .Where(x => !clientIds.Contains(x.ResourceId)
-                    || modifiedClientIds.Contains(x.ResourceId))
-        .Select(x => new {
-          x.ResourceId,
-          x.Name,
-          ResourceGroupId = x.ParentResourceId,
-          GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
-          ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
-          GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
-        })
-        .ToList();
-
-      var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
-      dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
-      dimClientDao.Save(newClients.Select(x => new DimClient {
+      var resourceDao = pm.ResourceDao;
+
+      var resources = resourceDao.GetAll().ToList();
+      var dimClients = dimClientDao.GetAllOnlineClients().ToList();
+
+      var onlineClients = dimClients.Where(x => resources.Select(y => y.ResourceId).Contains(x.ResourceId));
+      var addedResources = resources.Where(x => !dimClients.Select(y => y.ResourceId).Contains(x.ResourceId));
+      var removedResources = dimClients.Where(x => !resources.Select(y => y.ResourceId).Contains(x.ResourceId));
+
+      // set expiration time of removed resources
+      foreach(var r in removedResources) {
+        r.DateExpired = DateTime.Now;
+      }
+
+      // add new resources
+      dimClientDao.Save(addedResources.Select(x => new DimClient {
         ResourceId = x.ResourceId,
+        ParentResourceId = x.ParentResourceId,
         Name = x.Name,
-        ExpirationTime = null,
-        ResourceGroupId = x.ResourceGroupId,
-        GroupName = x.GroupName,
-        ResourceGroup2Id = x.ResourceGroup2Id,
-        GroupName2 = x.GroupName2
+        ResourceType = x.ResourceType,
+        DateCreated = DateTime.Now,
+        DateExpired = null
       }));
-    }
+
+      // expire client if its parent has changed and create a new entry
+      // otherwise perform "normal" update
+      foreach(var dimc in onlineClients) {
+        var r = resources.Where(x => x.ResourceId == dimc.ResourceId).SingleOrDefault();
+        if(r != null) {
+          if(dimc.ParentResourceId == null ? r.ParentResourceId != null : dimc.ParentResourceId != r.ParentResourceId) {
+            var now = DateTime.Now;
+            dimc.DateExpired = now;
+            dimClientDao.Save(new DimClient {
+              ResourceId = r.ResourceId,
+              ParentResourceId = r.ParentResourceId,
+              Name = r.Name,
+              ResourceType = r.ResourceType,
+              DateCreated = now,
+              DateExpired = null
+            });
+          } else {
+            dimc.Name = r.Name;
+          }
+        }
+      }
+    }
+
+    //// (1) for new slaves (not yet reported in Table DimClients) ...
+    //// and modified slaves (name or parent resource changed) a new DimClient-entry is created
+    //// (2) for already reported removed and modifid clients the expiration date is set
+    //private void UpdateDimClientsTableOld(PersistenceManager pm) {
+    //  var dimClientDao = pm.DimClientDao;
+    //  var slaveDao = pm.SlaveDao;
+    //  var slaves = slaveDao.GetAll();
+    //  var recentlyAddedClients = dimClientDao.GetAllOnlineClients();
+    //  var slaveIds = slaves.Select(x => x.ResourceId);
+
+    //  var removedClientIds = recentlyAddedClients
+    //    .Where(x => !slaveIds.Contains(x.ResourceId))
+    //    .Select(x => x.Id);
+    //  var modifiedClients =
+    //    from slave in slaves
+    //    join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
+    //    where (slave.Name != client.Name
+    //           || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
+    //           || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
+    //           || slave.ParentResourceId != client.ResourceGroupId
+    //           || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
+    //    select new {
+    //      SlaveId = slave.ResourceId,
+    //      ClientId = client.Id
+    //    };
+    //  var clientIds = dimClientDao.GetAllOnlineClients().Select(x => x.ResourceId);
+    //  var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
+    //  var newClients = slaves
+    //    .Where(x => !clientIds.Contains(x.ResourceId)
+    //                || modifiedClientIds.Contains(x.ResourceId))
+    //    .Select(x => new {
+    //      x.ResourceId,
+    //      x.Name,
+    //      ResourceGroupId = x.ParentResourceId,
+    //      GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
+    //      ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
+    //      GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
+    //    })
+    //    .ToList();
+
+    //  var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
+    //  dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
+    //  dimClientDao.Save(newClients.Select(x => new DimClient {
+    //    ResourceId = x.ResourceId,
+    //    Name = x.Name,
+    //    ExpirationTime = null,
+    //    ResourceGroupId = x.ResourceGroupId,
+    //    GroupName = x.GroupName,
+    //    ResourceGroup2Id = x.ResourceGroup2Id,
+    //    GroupName2 = x.GroupName2
+    //  }));
+    //}
+
 
     private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
@@ -222 +368 @@
       var newRawFactInfos =
         from s in slaveDao.GetAll()
-        join c in dimClientDao.GetActiveClients() on s.ResourceId equals c.ResourceId
+        join c in dimClientDao.GetAllOnlineSlaves() on s.ResourceId equals c.ResourceId
         join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
         from cf in joinCf.DefaultIfEmpty()
@@ -261 +407 @@
     }
 
-    private void UpdateTaskFactsTable(PersistenceManager pm) {
+    private void UpdateFactProjectInfoTable(DimTime newTime, PersistenceManager pm) {
+      var factProjectInfoDao = pm.FactProjectInfoDao;
+      var dimProjectDao = pm.DimProjectDao;
+      var projectDao = pm.ProjectDao;
+
+      var projectAvailabilityStats = projectDao.GetAvailabilityStatsPerProject();
+      var projectUsageStats = projectDao.GetUsageStatsPerProject();
+      var dimProjects = dimProjectDao.GetAllOnlineProjects().ToList();
+
+      factProjectInfoDao.Save(
+        from dimp in dimProjects
+        let aStats = projectAvailabilityStats.Where(x => x.ProjectId == dimp.ProjectId).SingleOrDefault()
+        let uStats = projectUsageStats.Where(x => x.ProjectId == dimp.ProjectId).SingleOrDefault()
+        select new FactProjectInfo {
+            ProjectId = dimp.Id,
+            DimTime = newTime,
+            NumTotalCores = aStats != null ? aStats.Cores : 0,
+            TotalMemory = aStats != null ? aStats.Memory : 0,
+            NumUsedCores = uStats != null ? uStats.Cores : 0,
+            UsedMemory = uStats != null ? uStats.Memory : 0
+          }
+        );
+    }
+
+    private void UpdateFactTaskTable(PersistenceManager pm) {
       var factTaskDao = pm.FactTaskDao;
       var taskDao = pm.TaskDao;
@@ -267 +437 @@
 
       var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
-      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
-        x.TaskId,
-        x.LastClientId
-      });
-
-      var newTasks =
+      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks();
+      //var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
+      //  x.TaskId,
+      //  x.LastClientId
+      //});
+
+      // query several properties for all new and not finished tasks
+      // in order to use them later either...
+      // (1) to update the fact task entry of not finished tasks
+      // (2) to insert a new fact task entry for new tasks
+      var newAndNotFinishedTasks =
         (from task in taskDao.GetAllChildTasks()
          let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
@@ -280 +455 @@
          join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
          from lastFact in lastFactPerTask.DefaultIfEmpty()
-         join client in dimClientDao.GetActiveClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
+         join client in dimClientDao.GetAllOnlineClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
          from client in clientsPerSlaveId.DefaultIfEmpty()
          select new {
@@ -296 +471 @@
          }).ToList();
 
-      //insert facts for new tasks
+      // (1) update data of already existing facts
+      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = true
+      foreach (var notFinishedFactTask in notFinishedFactTasks) {
+        var nfftUpdate = newAndNotFinishedTasks.Where(x => x.TaskId == notFinishedFactTask.TaskId).SingleOrDefault();
+        if(nfftUpdate != null) {
+          var taskData = CalculateFactTaskData(nfftUpdate.StateLogs);
+
+          notFinishedFactTask.StartTime = taskData.StartTime;
+          notFinishedFactTask.EndTime = taskData.EndTime;
+          notFinishedFactTask.LastClientId = nfftUpdate.LastClientId;
+          notFinishedFactTask.Priority = nfftUpdate.Priority;
+          notFinishedFactTask.CoresRequired = nfftUpdate.CoresRequired;
+          notFinishedFactTask.MemoryRequired = nfftUpdate.MemoryRequired;
+          notFinishedFactTask.NumCalculationRuns = taskData.CalculationRuns;
+          notFinishedFactTask.NumRetries = taskData.Retries;
+          notFinishedFactTask.WaitingTime = taskData.WaitingTime;
+          notFinishedFactTask.CalculatingTime = taskData.CalculatingTime;
+          notFinishedFactTask.TransferTime = taskData.TransferTime;
+          notFinishedFactTask.TaskState = nfftUpdate.State;
+          notFinishedFactTask.Exception = taskData.Exception;
+          notFinishedFactTask.InitialWaitingTime = taskData.InitialWaitingTime;
+        }
+      }
+
+      // (2) insert facts for new tasks
+      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = false
       factTaskDao.Save(
-        from x in newTasks
+        from x in newAndNotFinishedTasks
         where !x.NotFinishedTask
         let taskData = CalculateFactTaskData(x.StateLogs)
@@ -320 +520 @@
         });
 
-      //update data of already existing facts
-      foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
-        var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
-        if (ntc.Any()) {
-          var x = ntc.Single();
-          var taskData = CalculateFactTaskData(x.StateLogs);
-
-          notFinishedTask.StartTime = taskData.StartTime;
-          notFinishedTask.EndTime = taskData.EndTime;
-          notFinishedTask.LastClientId = x.LastClientId;
-          notFinishedTask.Priority = x.Priority;
-          notFinishedTask.CoresRequired = x.CoresRequired;
-          notFinishedTask.MemoryRequired = x.MemoryRequired;
-          notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
-          notFinishedTask.NumRetries = taskData.Retries;
-          notFinishedTask.WaitingTime = taskData.WaitingTime;
-          notFinishedTask.CalculatingTime = taskData.CalculatingTime;
-          notFinishedTask.TransferTime = taskData.TransferTime;
-          notFinishedTask.TaskState = x.State;
-          notFinishedTask.Exception = taskData.Exception;
-          notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
-        }
-      }
+
+      ////update data of already existing facts
+      //foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
+      //  var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
+      //  if (ntc.Any()) {
+      //    var x = ntc.Single();
+      //    var taskData = CalculateFactTaskData(x.StateLogs);
+
+      //    notFinishedTask.StartTime = taskData.StartTime;
+      //    notFinishedTask.EndTime = taskData.EndTime;
+      //    notFinishedTask.LastClientId = x.LastClientId;
+      //    notFinishedTask.Priority = x.Priority;
+      //    notFinishedTask.CoresRequired = x.CoresRequired;
+      //    notFinishedTask.MemoryRequired = x.MemoryRequired;
+      //    notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
+      //    notFinishedTask.NumRetries = taskData.Retries;
+      //    notFinishedTask.WaitingTime = taskData.WaitingTime;
+      //    notFinishedTask.CalculatingTime = taskData.CalculatingTime;
+      //    notFinishedTask.TransferTime = taskData.TransferTime;
+      //    notFinishedTask.TaskState = x.State;
+      //    notFinishedTask.Exception = taskData.Exception;
+      //    notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
+      //  }
+      //}
     }