Changeset 12516 for branches/HiveStatistics/sources/HeuristicLab.Services.Hive

Timestamp:

06/25/15 18:34:40 (9 years ago)

Author:

dglaser

Message:

#2388:

HeuristicLab.Services.Hive.DataAccess-3.3:

• updated daos
• changed statistics database schema
• updated HiveStatisticsGenerator

HeuristicLab.Services.WebApp.Statistics-3.3:

• added jobs, client and user page

Location:

branches/HiveStatistics/sources

Files:

• . (modified) (1 prop)
• HeuristicLab.Services.Hive/3.3/HiveDao.cs (modified) (1 diff)
• HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs (modified) (15 diffs)

branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveDao.cs

@@ -317 +317 @@
       exp.JobCount = jobs.Count();
       exp.CalculatingCount = jobs.Count(j => j.State == TaskState.Calculating);
-      exp.FinishedCount = jobs.Count(j => j.State == TaskState.Finished);
+      exp.FinishedCount = jobs.Count(j => j.State == TaskState.Finished || j.State == TaskState.Aborted || j.State == TaskState.Failed);
       return exp;
     }

branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

@@ -32 +32 @@
     private const string UnknownUserName = "Unknown";
     private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
+    private static readonly TaskState[] CompletedStates = { TaskState.Finished, TaskState.Aborted, TaskState.Failed };
 
     public void GenerateStatistics() {
@@ -42 +43 @@
         });
 
+        DimTime time = null;
         pm.UseTransaction(() => {
-          var newTime = UpdateDimTimeTable(pm);
-          if (newTime != null) {
+          time = UpdateDimTimeTable(pm);
+          pm.SubmitChanges();
+        });
+
+        if (time != null) {
+          pm.UseTransaction(() => {
+            UpdateFactClientInfoTable(time, pm);
             pm.SubmitChanges();
-            UpdateFactClientInfoTable(newTime, pm);
-            UpdateTaskFactsTable(newTime, pm);
+          });
+
+          pm.UseTransaction(() => {
+            UpdateTaskFactsTable(time, pm);
             try {
+              pm.SubmitChanges();
+              UpdateExistingDimJobs(pm);
               pm.SubmitChanges();
             }
@@ -58 +69 @@
                           Exception Details: {0}", e));
             }
-          }
-        });
+          });
+        }
       }
     }
@@ -97 +108 @@
       var dimJobDao = pm.DimJobDao;
       var jobDao = pm.JobDao;
+      var taskDao = pm.TaskDao;
       var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId);
       var newJobs = jobDao.GetAll()
@@ -104 +116 @@
           UserId = x.OwnerUserId,
           JobName = x.Name ?? string.Empty,
-          DateCreated = x.DateCreated
+          DateCreated = x.DateCreated,
+          TotalTasks = taskDao.GetAll().Count(y => y.JobId == x.JobId)
         })
         .ToList();
@@ -112 +125 @@
         UserId = x.UserId,
         UserName = GetUserName(x.UserId),
-        DateCreated = x.DateCreated
+        DateCreated = x.DateCreated,
+        TotalTasks = x.TotalTasks,
+        CompletedTasks = 0,
+        DateCompleted = null
       }));
+    }
+
+    private void UpdateExistingDimJobs(PersistenceManager pm) {
+      var dimJobDao = pm.DimJobDao;
+      var factTaskDao = pm.FactTaskDao;
+      foreach (var dimJob in dimJobDao.GetNotCompletedJobs()) {
+        var taskStates = factTaskDao.GetByJobId(dimJob.JobId)
+            .GroupBy(x => x.TaskState)
+            .Select(x => new {
+              State = x.Key,
+              Count = x.Count()
+            }).ToList();
+        int totalTasks = 0, completedTasks = 0;
+        foreach (var state in taskStates) {
+          totalTasks += state.Count;
+          if (CompletedStates.Contains(state.State)) {
+            completedTasks += state.Count;
+          }
+        }
+        if (totalTasks == completedTasks) {
+          dimJob.DateCompleted = factTaskDao.GetLastCompletedTaskFromJob(dimJob.JobId);
+        }
+        dimJob.TotalTasks = totalTasks;
+        dimJob.CompletedTasks = completedTasks;
+      }
     }
 
@@ -120 +161 @@
       var slaveDao = pm.SlaveDao;
       var slaves = slaveDao.GetAll();
-      var recentlyAddedClients = dimClientDao.GetRecentlyAddedClients();
+      var recentlyAddedClients = dimClientDao.GetActiveClients();
       var slaveIds = slaves.Select(x => x.ResourceId);
 
@@ -136 +177 @@
           ClientId = client.Id
         };
-      var clientIds = dimClientDao.GetAll().Select(x => x.ResourceId);
+      var clientIds = dimClientDao.GetActiveClients().Select(x => x.ResourceId);
       var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
       var newClients = slaves
@@ -145 +186 @@
           x.Name,
           ResourceGroupId = x.ParentResourceId,
-          ResourceGroup2Id = x.ParentResource.ParentResourceId
+          GroupName = x.ParentResource.Name,
+          ResourceGroup2Id = x.ParentResource.ParentResourceId,
+          GroupName2 = x.ParentResource.ParentResource.Name
         })
         .ToList();
@@ -156 +199 @@
         ExpirationTime = null,
         ResourceGroupId = x.ResourceGroupId,
-        ResourceGroup2Id = x.ResourceGroup2Id
+        GroupName = x.GroupName,
+        ResourceGroup2Id = x.ResourceGroup2Id,
+        GroupName2 = x.GroupName2
       }));
     }
@@ -167 +212 @@
       var newRawFactInfos =
         from s in slaveDao.GetAll()
-        join c in dimClientDao.GetRecentlyAddedClients() on s.ResourceId equals c.ResourceId
+        join c in dimClientDao.GetActiveClients() on s.ResourceId equals c.ResourceId
         join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
         from cf in joinCf.DefaultIfEmpty()
@@ -186 +231 @@
         from x in newRawFactInfos.ToList()
         let duration = x.LastFactTimestamp != null
-                       ? (newTime.Time - (DateTime)x.LastFactTimestamp).TotalMinutes
-                       : SmallestTimeSpan.TotalMinutes
+                       ? (int)(newTime.Time - (DateTime)x.LastFactTimestamp).TotalSeconds
+                       : (int)SmallestTimeSpan.TotalSeconds
         select new FactClientInfo {
           ClientId = x.ClientId,
@@ -193 +238 @@
           UserId = x.UserId,
           NumUsedCores = x.TotalCores - x.FreeCores,
-          NumTotalCores = x.TotalMemory,
+          NumTotalCores = x.TotalCores,
           UsedMemory = x.TotalMemory - x.FreeMemory,
           TotalMemory = x.TotalMemory,
@@ -199 +244 @@
           SlaveState = x.SlaveState,
           TotalTimeIdle = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0.0,
-          TotalTimeCalculating = x.SlaveState == SlaveState.Calculating ? duration : 0.0,
           TotalTimeUnavailable = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0.0,
           TotalTimeOffline = x.SlaveState == SlaveState.Offline ? duration : 0.0
@@ -226 +270 @@
         join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
         from lastFact in lastFactPerTask.DefaultIfEmpty()
-        join client in dimClientDao.GetRecentlyAddedClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
+        join client in dimClientDao.GetActiveClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
         from client in clientsPerSlaveId.DefaultIfEmpty()
         select new {