Changeset 12858 for branches/HiveStatistics
- Timestamp:
- 08/13/15 15:22:51 (9 years ago)
- Location:
- branches/HiveStatistics/sources
- Files:
-
- 16 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveJanitor.cs
r12789 r12858 55 55 56 56 public void RunCleanup() { 57 var pm = PersistenceManager; 57 58 while (!stop) { 58 59 try { 59 60 LogFactory.GetLogger(typeof(HiveJanitor).Namespace).Log("HiveJanitor: starting cleanup."); 60 61 bool cleanup = false; 61 using (var pm = PersistenceManager) { 62 63 64 65 66 67 68 69 70 71 72 } 62 63 var lifecycleDao = pm.LifecycleDao; 64 pm.UseTransaction(() => { 65 var lifecycle = lifecycleDao.GetLastLifecycle(); 66 if (lifecycle == null 67 || DateTime.Now - lifecycle.LastCleanup > HeuristicLab.Services.Hive.Properties.Settings.Default.CleanupInterval) { 68 lifecycleDao.UpdateLifecycle(); 69 cleanup = true; 70 } 71 pm.SubmitChanges(); 72 }, true); 73 73 74 if (cleanup) { 74 75 EventManager.Cleanup(); -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/AuthorizationManager.cs
r12691 r12858 52 52 public void AuthorizeForTask(Guid taskId, DT.Permission requiredPermission) { 53 53 if (ServiceLocator.Instance.RoleVerifier.IsInRole(HiveRoles.Slave)) return; // slave-users can access all tasks 54 using (var pm = PersistenceManager) { 55 var taskDao = pm.TaskDao; 56 pm.UseTransaction(() => { 57 var task = taskDao.GetById(taskId); 58 if (task == null) throw new SecurityException(NOT_AUTHORIZED); 59 AuthorizeJob(pm, task.JobId, requiredPermission); 60 }); 61 } 54 var pm = PersistenceManager; 55 var taskDao = pm.TaskDao; 56 pm.UseTransaction(() => { 57 var task = taskDao.GetById(taskId); 58 if (task == null) throw new SecurityException(NOT_AUTHORIZED); 59 AuthorizeJob(pm, task.JobId, requiredPermission); 60 }); 62 61 } 63 62 64 63 public void AuthorizeForJob(Guid jobId, DT.Permission requiredPermission) { 65 using (var pm = PersistenceManager) { 66 pm.UseTransaction(() => { 67 AuthorizeJob(pm, jobId, requiredPermission); 68 }); 69 } 64 var pm = PersistenceManager; 65 pm.UseTransaction(() => { 66 AuthorizeJob(pm, jobId, requiredPermission); 67 }); 70 68 } 71 69 72 70 public void AuthorizeForResourceAdministration(Guid resourceId) { 73 using (var pm = PersistenceManager) { 74 var resourceDao = pm.ResourceDao; 75 pm.UseTransaction(() => { 76 var resource = resourceDao.GetById(resourceId); 77 if (resource == null) throw new SecurityException(NOT_AUTHORIZED); 78 if (resource.OwnerUserId != UserManager.CurrentUserId 79 && !RoleVerifier.IsInRole(HiveRoles.Administrator)) { 80 throw new SecurityException(NOT_AUTHORIZED); 81 } 82 }); 83 } 71 var pm = PersistenceManager; 72 var resourceDao = pm.ResourceDao; 73 pm.UseTransaction(() => { 74 var resource = resourceDao.GetById(resourceId); 75 if (resource == null) throw new SecurityException(NOT_AUTHORIZED); 76 if (resource.OwnerUserId != UserManager.CurrentUserId 77 && !RoleVerifier.IsInRole(HiveRoles.Administrator)) { 78 throw new SecurityException(NOT_AUTHORIZED); 79 } 80 }); 84 81 } 85 82 -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/NewEventManager.cs
r12853 r12858 31 31 get { return ServiceLocator.Instance.PersistenceManager; } 32 32 } 33 private IAuthorizationManager AuthorizationManager {34 get { return ServiceLocator.Instance.AuthorizationManager; }35 }36 private ILogger Log {37 get { return LogFactory.GetLogger(this.GetType().Namespace); }38 }39 33 40 34 public void Cleanup() { 41 using (var pm = PersistenceManager) {42 43 44 45 46 47 48 35 var pm = PersistenceManager; 36 // same transactions as the old EventManager 37 pm.UseTransaction(() => { 38 SetTimeoutSlavesOffline(pm); 39 SetTimeoutTasksWaiting(pm); 40 DeleteObsoleteSlaves(pm); 41 pm.SubmitChanges(); 42 }); 49 43 50 pm.UseTransaction(() => { 51 FinishParentTasks(pm); 52 pm.SubmitChanges(); 53 }); 54 } 44 pm.UseTransaction(() => { 45 FinishParentTasks(pm); 46 pm.SubmitChanges(); 47 }); 55 48 } 56 49 -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/NewHeartbeatManager.cs
r12773 r12858 47 47 public List<MessageContainer> ProcessHeartbeat(Heartbeat heartbeat) { 48 48 List<MessageContainer> actions = new List<MessageContainer>(); 49 using (var pm = PersistenceManager) { 50 var slaveDao = pm.SlaveDao; 51 var taskDao = pm.TaskDao; 52 var slave = pm.UseTransaction(() => slaveDao.GetById(heartbeat.SlaveId)); 53 if (slave == null) { 54 actions.Add(new MessageContainer(MessageContainer.MessageType.SayHello)); 55 } else { 56 if (heartbeat.HbInterval != slave.HbInterval) { 57 actions.Add(new MessageContainer(MessageContainer.MessageType.NewHBInterval)); 58 } 59 if (slaveDao.SlaveHasToShutdownComputer(slave.ResourceId)) { 60 actions.Add(new MessageContainer(MessageContainer.MessageType.ShutdownComputer)); 61 } 62 // update slave data 63 slave.FreeCores = heartbeat.FreeCores; 64 slave.FreeMemory = heartbeat.FreeMemory; 65 slave.CpuUtilization = heartbeat.CpuUtilization; 66 slave.SlaveState = (heartbeat.JobProgress != null && heartbeat.JobProgress.Count > 0) 67 ? DA.SlaveState.Calculating 68 : DA.SlaveState.Idle; 69 slave.LastHeartbeat = DateTime.Now; 70 pm.UseTransaction(() => { 71 slave.IsAllowedToCalculate = slaveDao.SlaveIsAllowedToCalculate(slave.ResourceId); 72 pm.SubmitChanges(); 73 }); 74 75 // update task data 76 actions.AddRange(UpdateTasks(pm, heartbeat, slave.IsAllowedToCalculate)); 77 78 // assign new task 79 if (heartbeat.AssignJob && slave.IsAllowedToCalculate && heartbeat.FreeCores > 0) { 80 bool mutexAquired = false; 81 var mutex = new Mutex(false, MutexName); 82 try { 83 mutexAquired = mutex.WaitOne(Properties.Settings.Default.SchedulingPatience); 84 if (mutexAquired) { 85 var waitingTasks = pm.UseTransaction(() => taskDao.GetWaitingTasks(slave) 86 .Select(x => new TaskInfoForScheduler { 87 TaskId = x.TaskId, 88 JobId = x.JobId, 89 Priority = x.Priority 90 }) 91 .ToList() 92 ); 93 var availableTasks = TaskScheduler.Schedule(waitingTasks); 94 if (availableTasks.Any()) { 95 var task = availableTasks.First(); 96 AssignTask(pm, slave, task.TaskId); 97 actions.Add(new MessageContainer(MessageContainer.MessageType.CalculateTask, task.TaskId)); 98 } 99 } else { 100 LogFactory.GetLogger(this.GetType().Namespace).Log("HeartbeatManager: The mutex used for scheduling could not be aquired."); 49 var pm = PersistenceManager; 50 var slaveDao = pm.SlaveDao; 51 var taskDao = pm.TaskDao; 52 var slave = pm.UseTransaction(() => slaveDao.GetById(heartbeat.SlaveId)); 53 if (slave == null) { 54 actions.Add(new MessageContainer(MessageContainer.MessageType.SayHello)); 55 } else { 56 if (heartbeat.HbInterval != slave.HbInterval) { 57 actions.Add(new MessageContainer(MessageContainer.MessageType.NewHBInterval)); 58 } 59 if (slaveDao.SlaveHasToShutdownComputer(slave.ResourceId)) { 60 actions.Add(new MessageContainer(MessageContainer.MessageType.ShutdownComputer)); 61 } 62 // update slave data 63 slave.FreeCores = heartbeat.FreeCores; 64 slave.FreeMemory = heartbeat.FreeMemory; 65 slave.CpuUtilization = heartbeat.CpuUtilization; 66 slave.SlaveState = (heartbeat.JobProgress != null && heartbeat.JobProgress.Count > 0) 67 ? DA.SlaveState.Calculating 68 : DA.SlaveState.Idle; 69 slave.LastHeartbeat = DateTime.Now; 70 pm.UseTransaction(() => { 71 slave.IsAllowedToCalculate = slaveDao.SlaveIsAllowedToCalculate(slave.ResourceId); 72 pm.SubmitChanges(); 73 }); 74 75 // update task data 76 actions.AddRange(UpdateTasks(pm, heartbeat, slave.IsAllowedToCalculate)); 77 78 // assign new task 79 if (heartbeat.AssignJob && slave.IsAllowedToCalculate && heartbeat.FreeCores > 0) { 80 bool mutexAquired = false; 81 var mutex = new Mutex(false, MutexName); 82 try { 83 mutexAquired = mutex.WaitOne(Properties.Settings.Default.SchedulingPatience); 84 if (mutexAquired) { 85 var waitingTasks = pm.UseTransaction(() => taskDao.GetWaitingTasks(slave) 86 .Select(x => new TaskInfoForScheduler { 87 TaskId = x.TaskId, 88 JobId = x.JobId, 89 Priority = x.Priority 90 }) 91 .ToList() 92 ); 93 var availableTasks = TaskScheduler.Schedule(waitingTasks).ToArray(); 94 if (availableTasks.Any()) { 95 var task = availableTasks.First(); 96 AssignTask(pm, slave, task.TaskId); 97 actions.Add(new MessageContainer(MessageContainer.MessageType.CalculateTask, task.TaskId)); 101 98 } 99 } else { 100 LogFactory.GetLogger(this.GetType().Namespace).Log("HeartbeatManager: The mutex used for scheduling could not be aquired."); 102 101 } 103 catch (AbandonedMutexException) {104 LogFactory.GetLogger(this.GetType().Namespace).Log("HeartbeatManager: The mutex used for scheduling has been abandoned.");105 }106 catch (Exception ex) {107 LogFactory.GetLogger(this.GetType().Namespace).Log(string.Format("HeartbeatManager threw an exception in ProcessHeartbeat: {0}", ex));108 }109 finally {110 if (mutexAquired) mutex.ReleaseMutex();111 }102 } 103 catch (AbandonedMutexException) { 104 LogFactory.GetLogger(this.GetType().Namespace).Log("HeartbeatManager: The mutex used for scheduling has been abandoned."); 105 } 106 catch (Exception ex) { 107 LogFactory.GetLogger(this.GetType().Namespace).Log(string.Format("HeartbeatManager threw an exception in ProcessHeartbeat: {0}", ex)); 108 } 109 finally { 110 if (mutexAquired) mutex.ReleaseMutex(); 112 111 } 113 112 } -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Scheduler/NewRoundRobinTaskScheduler.cs
r12691 r12858 35 35 if (!tasks.Any()) return Enumerable.Empty<TaskInfoForScheduler>(); 36 36 37 using (var pm = PersistenceManager) {38 39 37 var pm = PersistenceManager; 38 var userPriorityDao = pm.UserPriorityDao; 39 var jobDao = pm.JobDao; 40 40 41 42 43 44 41 var userPriorities = pm.UseTransaction(() => userPriorityDao.GetAll() 42 .OrderBy(x => x.DateEnqueued) 43 .ToArray() 44 ); 45 45 46 47 48 49 50 51 52 53 54 55 56 46 var userIds = userPriorities.Select(x => x.UserId).ToList(); 47 var jobs = pm.UseTransaction(() => { 48 return jobDao.GetAll() 49 .Where(x => userIds.Contains(x.OwnerUserId)) 50 .Select(x => new { 51 Id = x.JobId, 52 DateCreated = x.DateCreated, 53 OwnerUserId = x.OwnerUserId 54 }) 55 .ToList(); 56 }); 57 57 58 59 60 61 62 63 58 var taskJobRelations = tasks.Join(jobs, 59 task => task.JobId, 60 job => job.Id, 61 (task, job) => new { Task = task, JobInfo = job }) 62 .OrderByDescending(x => x.Task.Priority) 63 .ToList(); 64 64 65 66 65 var scheduledTasks = new List<TaskInfoForScheduler>(); 66 int priorityIndex = 0; 67 67 68 68 if (count == 0 || count > taskJobRelations.Count) count = taskJobRelations.Count; 69 69 70 for (int i = 0; i < count; i++) { 71 var defaultEntry = taskJobRelations.First(); // search first task which is not included yet 72 var priorityEntries = taskJobRelations.Where(x => x.JobInfo.OwnerUserId == userPriorities[priorityIndex].UserId).ToArray(); // search for tasks with desired user priority 73 while (!priorityEntries.Any() && priorityIndex < userPriorities.Length - 1) { 70 for (int i = 0; i < count; i++) { 71 var defaultEntry = taskJobRelations.First(); // search first task which is not included yet 72 var priorityEntries = taskJobRelations.Where(x => x.JobInfo.OwnerUserId == userPriorities[priorityIndex].UserId).ToArray(); // search for tasks with desired user priority 73 while (!priorityEntries.Any() && priorityIndex < userPriorities.Length - 1) { 74 priorityIndex++; 75 priorityEntries = taskJobRelations.Where(x => x.JobInfo.OwnerUserId == userPriorities[priorityIndex].UserId).ToArray(); 76 } 77 if (priorityEntries.Any()) { // tasks with desired user priority found 78 var priorityEntry = priorityEntries.OrderByDescending(x => x.Task.Priority).ThenBy(x => x.JobInfo.DateCreated).First(); 79 if (defaultEntry.Task.Priority <= priorityEntry.Task.Priority) { 80 taskJobRelations.Remove(priorityEntry); 81 scheduledTasks.Add(priorityEntry.Task); 82 UpdateUserPriority(pm, userPriorities[priorityIndex]); 74 83 priorityIndex++; 75 priorityEntries = taskJobRelations.Where(x => x.JobInfo.OwnerUserId == userPriorities[priorityIndex].UserId).ToArray(); 76 } 77 if (priorityEntries.Any()) { // tasks with desired user priority found 78 var priorityEntry = priorityEntries.OrderByDescending(x => x.Task.Priority).ThenBy(x => x.JobInfo.DateCreated).First(); 79 if (defaultEntry.Task.Priority <= priorityEntry.Task.Priority) { 80 taskJobRelations.Remove(priorityEntry); 81 scheduledTasks.Add(priorityEntry.Task); 82 UpdateUserPriority(pm, userPriorities[priorityIndex]); 83 priorityIndex++; 84 } else { // there are other tasks with higher priorities 85 taskJobRelations.Remove(defaultEntry); 86 scheduledTasks.Add(defaultEntry.Task); 87 } 88 } else { 84 } else { // there are other tasks with higher priorities 89 85 taskJobRelations.Remove(defaultEntry); 90 86 scheduledTasks.Add(defaultEntry.Task); 91 87 } 92 if (priorityIndex >= (userPriorities.Length - 1)) priorityIndex = 0; 88 } else { 89 taskJobRelations.Remove(defaultEntry); 90 scheduledTasks.Add(defaultEntry.Task); 93 91 } 94 return scheduledTasks;92 if (priorityIndex >= (userPriorities.Length - 1)) priorityIndex = 0; 95 93 } 94 return scheduledTasks; 95 96 96 } 97 97 -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Maintenance/3.3/WebApi/FactClientInfoController.cs
r12777 r12858 36 36 37 37 public DT.FactClientInfo GetFactClientInfo(DateTime start, DateTime end) { 38 using (var pm = PersistenceManager) { 39 var factClientInfo = pm.FactClientInfoDao; 40 var query = factClientInfo.GetAll().Where(x => x.Time >= start && x.Time <= end); 41 return new DT.FactClientInfo { 42 Rows = query.Count(), 43 Clients = query.Select(x => x.ClientId).Distinct().Count() 44 }; 45 } 38 var pm = PersistenceManager; 39 var factClientInfo = pm.FactClientInfoDao; 40 var query = factClientInfo.GetAll().Where(x => x.Time >= start && x.Time <= end); 41 return new DT.FactClientInfo { 42 Rows = query.Count(), 43 Clients = query.Select(x => x.ClientId).Distinct().Count() 44 }; 46 45 } 47 46 48 47 public void Aggregate(DateTime start, DateTime end, int entries) { 49 using (var pm = PersistenceManager) { 50 var factClientInfoDao = pm.FactClientInfoDao; 51 var dimTimeDao = pm.DimTimeDao; 52 var clientIds = pm.UseTransaction(() => factClientInfoDao.GetAll() 53 .Where(x => x.Time >= start && x.Time <= end) 54 .Select(x => x.ClientId) 55 .Distinct() 56 ); 57 foreach (var id in clientIds) { 58 AggregateClient(pm, id, start, end, entries); 59 } 60 dimTimeDao.DeleteUnusedTimes(); 48 var pm = PersistenceManager; 49 var factClientInfoDao = pm.FactClientInfoDao; 50 var dimTimeDao = pm.DimTimeDao; 51 var clientIds = pm.UseTransaction(() => factClientInfoDao.GetAll() 52 .Where(x => x.Time >= start && x.Time <= end) 53 .Select(x => x.ClientId) 54 .Distinct() 55 ); 56 foreach (var id in clientIds) { 57 AggregateClient(pm, id, start, end, entries); 61 58 } 59 dimTimeDao.DeleteUnusedTimes(); 62 60 } 63 61 -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Maintenance/3.3/WebApi/FactTaskController.cs
r12761 r12858 35 35 36 36 public DT.JobPage GetJobs(DateTime start, DateTime end, int page, int size) { 37 using (var pm = PersistenceManager) { 38 var dimJobDao = pm.DimJobDao; 39 var factTaskDao = pm.FactTaskDao; 40 return pm.UseTransaction(() => { 41 var query = dimJobDao.GetAll().Where(x => x.DateCreated >= start && x.DateCreated <= end); 42 return new DT.JobPage { 43 TotalJobs = query.Count(), 44 Jobs = query.OrderBy(x => x.DateCreated) 45 .Skip((page - 1) * size) 46 .Take(size) 47 .Select(x => new DT.Job { 48 Id = x.JobId, 49 Name = x.JobName, 50 Username = x.UserName, 51 DateCreated = x.DateCreated, 52 Tasks = x.TotalTasks, 53 Aggregated = !(factTaskDao.GetAll().Count(y => y.JobId == x.JobId) > 1) 54 }) 55 .ToList() 56 }; 57 }); 58 } 37 var pm = PersistenceManager; 38 var dimJobDao = pm.DimJobDao; 39 var factTaskDao = pm.FactTaskDao; 40 return pm.UseTransaction(() => { 41 var query = dimJobDao.GetAll().Where(x => x.DateCreated >= start && x.DateCreated <= end); 42 return new DT.JobPage { 43 TotalJobs = query.Count(), 44 Jobs = query.OrderBy(x => x.DateCreated) 45 .Skip((page - 1) * size) 46 .Take(size) 47 .Select(x => new DT.Job { 48 Id = x.JobId, 49 Name = x.JobName, 50 Username = x.UserName, 51 DateCreated = x.DateCreated, 52 Tasks = x.TotalTasks, 53 Aggregated = !(factTaskDao.GetAll().Count(y => y.JobId == x.JobId) > 1) 54 }) 55 .ToList() 56 }; 57 }); 59 58 } 60 59 61 60 [HttpPost] 62 61 public void AggregateJob(Guid id) { 63 using (var pm = PersistenceManager) { 64 AggregateJob(pm, id); 65 } 62 var pm = PersistenceManager; 63 AggregateJob(pm, id); 66 64 } 67 65 68 66 [HttpPost] 69 67 public void AggregateAllJobs(DateTime start, DateTime end) { 70 using (var pm = PersistenceManager) { 71 var dimJobDao = pm.DimJobDao; 72 var jobIds = pm.UseTransaction(() => dimJobDao.GetAll() 73 .Where(x => x.DateCreated >= start && x.DateCreated <= end) 74 .Select(x => x.JobId) 75 .ToList() 76 ); 77 foreach (var id in jobIds) { 78 AggregateJob(pm, id); 79 } 68 var pm = PersistenceManager; 69 var dimJobDao = pm.DimJobDao; 70 var jobIds = pm.UseTransaction(() => dimJobDao.GetAll() 71 .Where(x => x.DateCreated >= start && x.DateCreated <= end) 72 .Select(x => x.JobId) 73 .ToList() 74 ); 75 foreach (var id in jobIds) { 76 AggregateJob(pm, id); 80 77 } 81 78 } -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Maintenance/3.3/WebApi/PluginController.cs
r12769 r12858 34 34 35 35 public DT.PluginPage GetUnusedPlugins(int page, int size) { 36 using (var pm = PersistenceManager) { 37 var pluginDao = pm.PluginDao; 38 var requiredPluginDao = pm.RequiredPluginDao; 39 var taskDao = pm.TaskDao; 40 return pm.UseTransaction(() => { 41 var taskIds = taskDao.GetAll().Select(x => x.TaskId); 42 var usedPluginIds = requiredPluginDao.GetAll() 43 .Where(x => taskIds.Contains(x.TaskId)) 44 .Select(x => x.PluginId) 45 .Distinct(); 46 var query = pluginDao.GetAll().Where(x => !usedPluginIds.Any(y => y == x.PluginId)); 47 return new DT.PluginPage { 48 TotalPlugins = query.Count(), 49 Plugins = query.OrderBy(x => x.DateCreated).ThenBy(x => x.Name) 50 .Skip((page - 1) * size) 51 .Take(size).Select(x => new DT.Plugin { 52 Id = x.PluginId, 53 Name = x.Name, 54 Version = x.Version, 55 DateCreated = x.DateCreated 56 }) 57 .ToList() 58 }; 59 }); 60 } 36 var pm = PersistenceManager; 37 var pluginDao = pm.PluginDao; 38 var requiredPluginDao = pm.RequiredPluginDao; 39 var taskDao = pm.TaskDao; 40 return pm.UseTransaction(() => { 41 var taskIds = taskDao.GetAll().Select(x => x.TaskId); 42 var usedPluginIds = requiredPluginDao.GetAll() 43 .Where(x => taskIds.Contains(x.TaskId)) 44 .Select(x => x.PluginId) 45 .Distinct(); 46 var query = pluginDao.GetAll().Where(x => !usedPluginIds.Any(y => y == x.PluginId)); 47 return new DT.PluginPage { 48 TotalPlugins = query.Count(), 49 Plugins = query.OrderBy(x => x.DateCreated).ThenBy(x => x.Name) 50 .Skip((page - 1) * size) 51 .Take(size).Select(x => new DT.Plugin { 52 Id = x.PluginId, 53 Name = x.Name, 54 Version = x.Version, 55 DateCreated = x.DateCreated 56 }) 57 .ToList() 58 }; 59 }); 61 60 } 62 61 63 62 [HttpPost] 64 63 public void DeletePlugin(Guid id) { 65 using (var pm = PersistenceManager) { 66 var pluginDao = pm.PluginDao; 67 pm.UseTransaction(() => { 68 pluginDao.Delete(id); 69 pm.SubmitChanges(); 70 }); 71 } 64 var pm = PersistenceManager; 65 var pluginDao = pm.PluginDao; 66 pm.UseTransaction(() => { 67 pluginDao.Delete(id); 68 pm.SubmitChanges(); 69 }); 72 70 } 73 71 74 72 [HttpPost] 75 73 public void DeleteUnusedPlugins() { 76 using (var pm = PersistenceManager) { 77 var pluginDao = pm.PluginDao; 78 pm.UseTransaction(() => { 79 pluginDao.DeleteUnusedPlugins(); 80 pm.SubmitChanges(); 81 }); 82 } 74 var pm = PersistenceManager; 75 var pluginDao = pm.PluginDao; 76 pm.UseTransaction(() => { 77 pluginDao.DeleteUnusedPlugins(); 78 pm.SubmitChanges(); 79 }); 83 80 } 84 81 } -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Maintenance/3.3/WebApi/SpaceUsageController.cs
r12761 r12858 51 51 "UserPriority" 52 52 }; 53 using (var pm = PersistenceManager) {54 53 var pm = PersistenceManager; 54 return pm.UseTransaction(() => ( 55 55 from table in tables 56 56 select pm.GetTableInformation(table) into tableInformation … … 59 59 ).ToList() 60 60 ); 61 }62 61 } 63 62 … … 71 70 "statistics.FactTask" 72 71 }; 73 using (var pm = PersistenceManager) {74 72 var pm = PersistenceManager; 73 return pm.UseTransaction(() => ( 75 74 from table in tables 76 75 select pm.GetTableInformation(table) into tableInformation … … 79 78 ).ToList() 80 79 ); 81 }82 80 } 83 81 -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Statistics/3.3/WebApi/ClientController.cs
r12776 r12858 38 38 39 39 public DT.ClientDetails GetClientDetails(Guid id) { 40 using (var pm = PersistenceManager) { 41 var dimClientDao = pm.DimClientDao; 42 var factClientInfoDao = pm.FactClientInfoDao; 43 var factTaskDao = pm.FactTaskDao; 44 return pm.UseTransaction(() => { 45 var timeData = factClientInfoDao.GetByClientId(id) 46 .GroupBy(x => x.ClientId) 47 .Select(x => new { 48 TotalIdleTime = x.Sum(y => y.IdleTime), 49 TotalOfflineTime = x.Sum(y => y.OfflineTime), 50 TotalUnavailableTime = x.Sum(y => y.UnavailableTime) 51 }).FirstOrDefault(); 52 var taskTimeData = factTaskDao.GetByClientId(id) 53 .GroupBy(x => x.LastClientId) 54 .Select(x => new { 55 TotalCalculatingTime = x.Sum(y => y.CalculatingTime), 56 TotalTransferringTime = x.Sum(y => y.TransferTime) 57 }).FirstOrDefault(); 58 var startDate = factClientInfoDao.GetByClientId(id) 59 .Where(x => x.SlaveState == SlaveState.Offline) 40 var pm = PersistenceManager; 41 var dimClientDao = pm.DimClientDao; 42 var factClientInfoDao = pm.FactClientInfoDao; 43 var factTaskDao = pm.FactTaskDao; 44 return pm.UseTransaction(() => { 45 var timeData = factClientInfoDao.GetByClientId(id) 46 .GroupBy(x => x.ClientId) 47 .Select(x => new { 48 TotalIdleTime = x.Sum(y => y.IdleTime), 49 TotalOfflineTime = x.Sum(y => y.OfflineTime), 50 TotalUnavailableTime = x.Sum(y => y.UnavailableTime) 51 }).FirstOrDefault(); 52 var taskTimeData = factTaskDao.GetByClientId(id) 53 .GroupBy(x => x.LastClientId) 54 .Select(x => new { 55 TotalCalculatingTime = x.Sum(y => y.CalculatingTime), 56 TotalTransferringTime = x.Sum(y => y.TransferTime) 57 }).FirstOrDefault(); 58 var startDate = factClientInfoDao.GetByClientId(id) 59 .Where(x => x.SlaveState == SlaveState.Offline) 60 .OrderByDescending(x => x.Time) 61 .Select(x => x.Time).FirstOrDefault(); 62 long upTime = 0; 63 if (startDate == default(DateTime)) { 64 startDate = factClientInfoDao.GetByClientId(id) 60 65 .OrderByDescending(x => x.Time) 61 .Select(x => x.Time).FirstOrDefault(); 62 long upTime = 0; 63 if (startDate == default(DateTime)) { 64 startDate = factClientInfoDao.GetByClientId(id) 65 .OrderByDescending(x => x.Time) 66 .Select(x => x.Time) 67 .FirstOrDefault(); 68 } 69 if (startDate != default(DateTime)) { 70 upTime = (long)(DateTime.Now - startDate).TotalSeconds; 71 } 72 return (from client in dimClientDao.GetAll().Where(x => x.Id == id) 73 join info in factClientInfoDao.GetAll() 74 on client.Id equals info.ClientId into clientInfoJoin 75 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 76 let offline = (client.ExpirationTime != null || clientInfo.SlaveState == SlaveState.Offline) 77 select new DT.ClientDetails { 78 Id = client.Id, 79 Name = client.Name, 80 TotalCores = clientInfo.NumTotalCores, 81 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 82 TotalMemory = clientInfo.TotalMemory, 83 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 84 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 85 State = offline ? SlaveState.Offline.ToString() : clientInfo.SlaveState.ToString(), 86 LastUpdate = clientInfo.Time, 87 GroupId = client.ResourceGroupId, 88 GroupName = client.GroupName, 89 UpTime = offline ? 0 : upTime, 90 TotalUnavailableTime = timeData != null ? timeData.TotalUnavailableTime : 0, 91 TotalCalculatingTime = taskTimeData != null ? taskTimeData.TotalCalculatingTime : 0, 92 TotalIdleTime = timeData != null ? timeData.TotalIdleTime : 0, 93 TotalOfflineTime = timeData != null ? timeData.TotalOfflineTime : 0, 94 TotalTransferringTime = taskTimeData != null ? taskTimeData.TotalTransferringTime : 0, 95 TasksStates = factTaskDao.GetByClientId(id) 96 .GroupBy(x => x.TaskState) 97 .Select(x => new DT.TaskStateCount { 98 State = x.Key.ToString(), 99 Count = x.Count() 100 }).ToList(), 101 Users = factTaskDao.GetAll() 102 .Where(x => x.LastClientId == id) 103 .Select(x => new DT.User { 104 Id = x.DimJob.UserId, 105 Name = x.DimJob.UserName 106 }) 107 .Distinct() 108 .ToList() 109 }) 110 .FirstOrDefault(); 111 }); 112 } 66 .Select(x => x.Time) 67 .FirstOrDefault(); 68 } 69 if (startDate != default(DateTime)) { 70 upTime = (long)(DateTime.Now - startDate).TotalSeconds; 71 } 72 return (from client in dimClientDao.GetAll().Where(x => x.Id == id) 73 join info in factClientInfoDao.GetAll() 74 on client.Id equals info.ClientId into clientInfoJoin 75 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 76 let offline = (client.ExpirationTime != null || clientInfo.SlaveState == SlaveState.Offline) 77 select new DT.ClientDetails { 78 Id = client.Id, 79 Name = client.Name, 80 TotalCores = clientInfo.NumTotalCores, 81 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 82 TotalMemory = clientInfo.TotalMemory, 83 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 84 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 85 State = offline ? SlaveState.Offline.ToString() : clientInfo.SlaveState.ToString(), 86 LastUpdate = clientInfo.Time, 87 GroupId = client.ResourceGroupId, 88 GroupName = client.GroupName, 89 UpTime = offline ? 0 : upTime, 90 TotalUnavailableTime = timeData != null ? timeData.TotalUnavailableTime : 0, 91 TotalCalculatingTime = taskTimeData != null ? taskTimeData.TotalCalculatingTime : 0, 92 TotalIdleTime = timeData != null ? timeData.TotalIdleTime : 0, 93 TotalOfflineTime = timeData != null ? timeData.TotalOfflineTime : 0, 94 TotalTransferringTime = taskTimeData != null ? taskTimeData.TotalTransferringTime : 0, 95 TasksStates = factTaskDao.GetByClientId(id) 96 .GroupBy(x => x.TaskState) 97 .Select(x => new DT.TaskStateCount { 98 State = x.Key.ToString(), 99 Count = x.Count() 100 }).ToList(), 101 Users = factTaskDao.GetAll() 102 .Where(x => x.LastClientId == id) 103 .Select(x => new DT.User { 104 Id = x.DimJob.UserId, 105 Name = x.DimJob.UserName 106 }) 107 .Distinct() 108 .ToList() 109 }) 110 .FirstOrDefault(); 111 }); 113 112 } 114 113 115 114 public DT.ClientPage GetClients(int page, int size, bool expired = false) { 116 using (var pm = PersistenceManager) { 117 var dimClientDao = pm.DimClientDao; 118 var factClientInfoDao = pm.FactClientInfoDao; 119 return pm.UseTransaction(() => { 120 var clients = expired ? dimClientDao.GetExpiredClients() : dimClientDao.GetActiveClients(); 121 var query = (from client in clients 122 join info in factClientInfoDao.GetAll() 123 on client.Id equals info.ClientId into clientInfoJoin 124 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 125 let offline = (expired || clientInfo.SlaveState == SlaveState.Offline) 126 select new DT.Client { 127 Id = client.Id, 128 Name = client.Name, 129 TotalCores = clientInfo.NumTotalCores, 130 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 131 TotalMemory = clientInfo.TotalMemory, 132 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 133 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 134 State = offline ? SlaveState.Offline.ToString() : clientInfo.SlaveState.ToString(), 135 GroupId = client.ResourceGroupId, 136 GroupName = client.GroupName, 137 IsAllowedToCalculate = clientInfo.IsAllowedToCalculate 138 }); 139 return new DT.ClientPage { 140 TotalClients = query.Count(), 141 Clients = query.Skip((page - 1) * size) 142 .Take(size) 143 .ToList() 144 }; 145 }); 146 } 115 var pm = PersistenceManager; 116 var dimClientDao = pm.DimClientDao; 117 var factClientInfoDao = pm.FactClientInfoDao; 118 return pm.UseTransaction(() => { 119 var clients = expired ? dimClientDao.GetExpiredClients() : dimClientDao.GetActiveClients(); 120 var query = (from client in clients 121 join info in factClientInfoDao.GetAll() 122 on client.Id equals info.ClientId into clientInfoJoin 123 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 124 let offline = (expired || clientInfo.SlaveState == SlaveState.Offline) 125 select new DT.Client { 126 Id = client.Id, 127 Name = client.Name, 128 TotalCores = clientInfo.NumTotalCores, 129 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 130 TotalMemory = clientInfo.TotalMemory, 131 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 132 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 133 State = offline ? SlaveState.Offline.ToString() : clientInfo.SlaveState.ToString(), 134 GroupId = client.ResourceGroupId, 135 GroupName = client.GroupName, 136 IsAllowedToCalculate = clientInfo.IsAllowedToCalculate 137 }); 138 return new DT.ClientPage { 139 TotalClients = query.Count(), 140 Clients = query.Skip((page - 1) * size) 141 .Take(size) 142 .ToList() 143 }; 144 }); 147 145 } 148 146 … … 155 153 increment += 5; 156 154 } 157 using (var pm = PersistenceManager) { 158 var factClientInfo = pm.FactClientInfoDao; 159 var clientInfos = factClientInfo.GetByClientId(id) 160 .Where(x => x.Time >= start && x.Time <= end) 161 .OrderBy(x => x.Time) 162 .ToList(); 163 var clientStatus = new DT.ClientStatus { 164 CpuUtilization = 0, 165 TotalCores = 0, 166 UsedCores = 0, 167 TotalMemory = 0, 168 UsedMemory = 0 155 var pm = PersistenceManager; 156 var factClientInfo = pm.FactClientInfoDao; 157 var clientInfos = factClientInfo.GetByClientId(id) 158 .Where(x => x.Time >= start && x.Time <= end) 159 .OrderBy(x => x.Time) 160 .ToList(); 161 var clientStatus = new DT.ClientStatus { 162 CpuUtilization = 0, 163 TotalCores = 0, 164 UsedCores = 0, 165 TotalMemory = 0, 166 UsedMemory = 0 167 }; 168 int i = 1; 169 foreach (var clientInfo in clientInfos) { 170 clientStatus.CpuUtilization += clientInfo.CpuUtilization; 171 clientStatus.TotalCores += clientInfo.NumTotalCores; 172 clientStatus.UsedCores += clientInfo.NumUsedCores; 173 clientStatus.TotalMemory += clientInfo.TotalMemory; 174 clientStatus.UsedMemory += clientInfo.UsedMemory; 175 if (i >= increment) { 176 clientStatus.Timestamp = JavascriptUtils.ToTimestamp(clientInfo.Time); 177 clientStatus.CpuUtilization /= i; 178 clientStatus.TotalCores /= i; 179 clientStatus.UsedCores /= i; 180 clientStatus.TotalMemory /= i; 181 clientStatus.UsedMemory /= i; 182 yield return clientStatus; 183 clientStatus = new DT.ClientStatus { 184 CpuUtilization = 0, 185 TotalCores = 0, 186 UsedCores = 0, 187 TotalMemory = 0, 188 UsedMemory = 0 189 }; 190 i = 1; 191 } else { 192 ++i; 193 } 194 } 195 } 196 197 public DT.ClientPage GetClientsByGroupId(Guid id, int page, int size, bool expired = false) { 198 var pm = PersistenceManager; 199 var dimClientDao = pm.DimClientDao; 200 var factClientInfoDao = pm.FactClientInfoDao; 201 return pm.UseTransaction(() => { 202 var clients = expired ? dimClientDao.GetExpiredClients() : dimClientDao.GetActiveClients(); 203 clients = clients.Where(x => x.ResourceGroupId == id); 204 var query = (from client in clients 205 join info in factClientInfoDao.GetAll() 206 on client.Id equals info.ClientId into clientInfoJoin 207 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 208 let offline = (expired || clientInfo.SlaveState == SlaveState.Offline) 209 select new DT.Client { 210 Id = client.Id, 211 Name = client.Name, 212 TotalCores = clientInfo.NumTotalCores, 213 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 214 TotalMemory = clientInfo.TotalMemory, 215 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 216 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 217 State = offline ? SlaveState.Offline.ToString() : clientInfo.SlaveState.ToString(), 218 GroupId = client.ResourceGroupId, 219 GroupName = client.GroupName, 220 IsAllowedToCalculate = clientInfo.IsAllowedToCalculate 221 }); 222 return new DT.ClientPage { 223 TotalClients = query.Count(), 224 Clients = query.Skip((page - 1) * size) 225 .Take(size) 226 .ToList() 169 227 }; 170 int i = 1; 171 foreach (var clientInfo in clientInfos) { 172 clientStatus.CpuUtilization += clientInfo.CpuUtilization; 173 clientStatus.TotalCores += clientInfo.NumTotalCores; 174 clientStatus.UsedCores += clientInfo.NumUsedCores; 175 clientStatus.TotalMemory += clientInfo.TotalMemory; 176 clientStatus.UsedMemory += clientInfo.UsedMemory; 177 if (i >= increment) { 178 clientStatus.Timestamp = JavascriptUtils.ToTimestamp(clientInfo.Time); 179 clientStatus.CpuUtilization /= i; 180 clientStatus.TotalCores /= i; 181 clientStatus.UsedCores /= i; 182 clientStatus.TotalMemory /= i; 183 clientStatus.UsedMemory /= i; 184 yield return clientStatus; 185 clientStatus = new DT.ClientStatus { 186 CpuUtilization = 0, 187 TotalCores = 0, 188 UsedCores = 0, 189 TotalMemory = 0, 190 UsedMemory = 0 191 }; 192 i = 1; 193 } else { 194 ++i; 195 } 196 } 197 } 198 } 199 200 public DT.ClientPage GetClientsByGroupId(Guid id, int page, int size, bool expired = false) { 201 using (var pm = PersistenceManager) { 202 var dimClientDao = pm.DimClientDao; 203 var factClientInfoDao = pm.FactClientInfoDao; 204 return pm.UseTransaction(() => { 205 var clients = expired ? dimClientDao.GetExpiredClients() : dimClientDao.GetActiveClients(); 206 clients = clients.Where(x => x.ResourceGroupId == id); 207 var query = (from client in clients 208 join info in factClientInfoDao.GetAll() 209 on client.Id equals info.ClientId into clientInfoJoin 210 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 211 let offline = (expired || clientInfo.SlaveState == SlaveState.Offline) 212 select new DT.Client { 213 Id = client.Id, 214 Name = client.Name, 215 TotalCores = clientInfo.NumTotalCores, 216 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 217 TotalMemory = clientInfo.TotalMemory, 218 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 219 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 220 State = offline ? SlaveState.Offline.ToString() : clientInfo.SlaveState.ToString(), 221 GroupId = client.ResourceGroupId, 222 GroupName = client.GroupName, 223 IsAllowedToCalculate = clientInfo.IsAllowedToCalculate 224 }); 225 return new DT.ClientPage { 226 TotalClients = query.Count(), 227 Clients = query.Skip((page - 1) * size) 228 .Take(size) 229 .ToList() 230 }; 231 }); 232 } 228 }); 233 229 } 234 230 … … 241 237 increment += 5; 242 238 } 243 using (var pm = PersistenceManager) { 244 var factClientInfo = pm.FactClientInfoDao; 245 var dimClientDao = pm.DimClientDao; 246 var clientInfos = factClientInfo.GetAll() 247 .Where(x => x.Time >= start && x.Time <= end) 248 .OrderBy(x => x.Time) 249 .Join(dimClientDao.GetAll(), 250 fact => fact.ClientId, 251 client => client.Id, 252 (fact, client) => new { 253 client.ResourceGroupId, 254 fact.Time, 255 fact.CpuUtilization, 256 fact.NumTotalCores, 257 fact.NumUsedCores, 258 fact.TotalMemory, 259 fact.UsedMemory 260 }) 261 .Where(x => x.ResourceGroupId == id) 262 .ToList(); 263 var clientStatus = new DT.ClientStatus { 264 CpuUtilization = 0, 265 TotalCores = 0, 266 UsedCores = 0, 267 TotalMemory = 0, 268 UsedMemory = 0 269 }; 270 int i = 1; 271 foreach (var clientInfo in clientInfos) { 272 clientStatus.CpuUtilization += clientInfo.CpuUtilization; 273 clientStatus.TotalCores += clientInfo.NumTotalCores; 274 clientStatus.UsedCores += clientInfo.NumUsedCores; 275 clientStatus.TotalMemory += clientInfo.TotalMemory; 276 clientStatus.UsedMemory += clientInfo.UsedMemory; 277 if (i >= increment) { 278 clientStatus.Timestamp = JavascriptUtils.ToTimestamp(clientInfo.Time); 279 clientStatus.CpuUtilization /= i; 280 clientStatus.TotalCores /= i; 281 clientStatus.UsedCores /= i; 282 clientStatus.TotalMemory /= i; 283 clientStatus.UsedMemory /= i; 284 yield return clientStatus; 285 clientStatus = new DT.ClientStatus { 286 CpuUtilization = 0, 287 TotalCores = 0, 288 UsedCores = 0, 289 TotalMemory = 0, 290 UsedMemory = 0 291 }; 292 i = 1; 293 } else { 294 ++i; 295 } 239 var pm = PersistenceManager; 240 var factClientInfo = pm.FactClientInfoDao; 241 var dimClientDao = pm.DimClientDao; 242 var clientInfos = factClientInfo.GetAll() 243 .Where(x => x.Time >= start && x.Time <= end) 244 .OrderBy(x => x.Time) 245 .Join(dimClientDao.GetAll(), 246 fact => fact.ClientId, 247 client => client.Id, 248 (fact, client) => new { 249 client.ResourceGroupId, 250 fact.Time, 251 fact.CpuUtilization, 252 fact.NumTotalCores, 253 fact.NumUsedCores, 254 fact.TotalMemory, 255 fact.UsedMemory 256 }) 257 .Where(x => x.ResourceGroupId == id) 258 .ToList(); 259 var clientStatus = new DT.ClientStatus { 260 CpuUtilization = 0, 261 TotalCores = 0, 262 UsedCores = 0, 263 TotalMemory = 0, 264 UsedMemory = 0 265 }; 266 int i = 1; 267 foreach (var clientInfo in clientInfos) { 268 clientStatus.CpuUtilization += clientInfo.CpuUtilization; 269 clientStatus.TotalCores += clientInfo.NumTotalCores; 270 clientStatus.UsedCores += clientInfo.NumUsedCores; 271 clientStatus.TotalMemory += clientInfo.TotalMemory; 272 clientStatus.UsedMemory += clientInfo.UsedMemory; 273 if (i >= increment) { 274 clientStatus.Timestamp = JavascriptUtils.ToTimestamp(clientInfo.Time); 275 clientStatus.CpuUtilization /= i; 276 clientStatus.TotalCores /= i; 277 clientStatus.UsedCores /= i; 278 clientStatus.TotalMemory /= i; 279 clientStatus.UsedMemory /= i; 280 yield return clientStatus; 281 clientStatus = new DT.ClientStatus { 282 CpuUtilization = 0, 283 TotalCores = 0, 284 UsedCores = 0, 285 TotalMemory = 0, 286 UsedMemory = 0 287 }; 288 i = 1; 289 } else { 290 ++i; 296 291 } 297 292 } -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Statistics/3.3/WebApi/ExceptionController.cs
r12562 r12858 35 35 36 36 public DT.ExceptionPage GetExceptions(int page, int size) { 37 using (var pm = PersistenceManager) { 38 var dimClientDao = pm.DimClientDao; 39 var dimJobDao = pm.DimJobDao; 40 var factTaskDao = pm.FactTaskDao; 41 var tasks = factTaskDao.GetTasksWithException(); 42 return pm.UseTransaction(() => new DT.ExceptionPage { 43 TotalExceptions = tasks.Count(), 44 Exceptions = (from factTask in tasks 45 join dimJob in dimJobDao.GetAll() on factTask.JobId equals dimJob.JobId 46 join dimClient in dimClientDao.GetAll() on factTask.LastClientId equals 47 dimClient.Id into taskClientJoin 48 from a in taskClientJoin.DefaultIfEmpty() 49 let endTime = factTask.EndTime ?? DateTime.Now 50 select new DT.Exception { 51 TaskId = factTask.TaskId, 52 JobId = factTask.JobId, 53 JobName = dimJob.JobName, 54 UserId = dimJob.UserId, 55 UserName = dimJob.UserName, 56 ClientId = factTask.LastClientId ?? default(Guid), 57 ClientName = a != null ? a.Name : string.Empty, 58 Date = endTime, 59 Details = factTask.Exception 60 }) 61 .OrderByDescending(x => x.Date) 62 .Skip((page - 1) * size) 63 .Take(size) 64 .ToList() 65 }); 66 } 37 var pm = PersistenceManager; 38 var dimClientDao = pm.DimClientDao; 39 var dimJobDao = pm.DimJobDao; 40 var factTaskDao = pm.FactTaskDao; 41 var tasks = factTaskDao.GetTasksWithException(); 42 return pm.UseTransaction(() => new DT.ExceptionPage { 43 TotalExceptions = tasks.Count(), 44 Exceptions = (from factTask in tasks 45 join dimJob in dimJobDao.GetAll() on factTask.JobId equals dimJob.JobId 46 join dimClient in dimClientDao.GetAll() on factTask.LastClientId equals 47 dimClient.Id into taskClientJoin 48 from a in taskClientJoin.DefaultIfEmpty() 49 let endTime = factTask.EndTime ?? DateTime.Now 50 select new DT.Exception { 51 TaskId = factTask.TaskId, 52 JobId = factTask.JobId, 53 JobName = dimJob.JobName, 54 UserId = dimJob.UserId, 55 UserName = dimJob.UserName, 56 ClientId = factTask.LastClientId ?? default(Guid), 57 ClientName = a != null ? a.Name : string.Empty, 58 Date = endTime, 59 Details = factTask.Exception 60 }) 61 .OrderByDescending(x => x.Date) 62 .Skip((page - 1) * size) 63 .Take(size) 64 .ToList() 65 }); 67 66 } 68 67 } -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Statistics/3.3/WebApi/GroupController.cs
r12562 r12858 35 35 36 36 public DT.GroupDetails GetGroupDetails(Guid id) { 37 using (var pm = PersistenceManager) {38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 37 var pm = PersistenceManager; 38 var dimClientDao = pm.DimClientDao; 39 var factClientInfoDao = pm.FactClientInfoDao; 40 var factTaskDao = pm.FactTaskDao; 41 return pm.UseTransaction(() => { 42 var clientTimeData = factClientInfoDao.GetAll() 43 .Join(dimClientDao.GetAll(), x => x.ClientId, y => y.Id, (x, y) => new { 44 y.ResourceGroupId, 45 x.IdleTime, 46 x.OfflineTime, 47 x.UnavailableTime 48 }) 49 .Where(x => x.ResourceGroupId == id) 50 .GroupBy(x => x.ResourceGroupId) 51 .Select(x => new { 52 TotalIdleTime = x.Sum(y => y.IdleTime), 53 TotalOfflineTime = x.Sum(y => y.OfflineTime), 54 TotalUnavailableTime = x.Sum(y => y.UnavailableTime) 55 }) 56 .FirstOrDefault(); 57 57 58 var taskTimeData = factTaskDao.GetByGroupId(id) 59 .Select(x => new { 60 id, 61 x.CalculatingTime, 62 x.TransferTime 63 }) 64 .GroupBy(x => x.id) 65 .Select(x => new { 66 CalculatingTime = x.Sum(y => y.CalculatingTime), 67 TransferTime = x.Sum(y => y.TransferTime) 68 }) 69 .FirstOrDefault(); 70 return (from client in dimClientDao.GetActiveClients().Where(x => x.ResourceGroupId == id) 58 var taskTimeData = factTaskDao.GetByGroupId(id) 59 .Select(x => new { 60 id, 61 x.CalculatingTime, 62 x.TransferTime 63 }) 64 .GroupBy(x => x.id) 65 .Select(x => new { 66 CalculatingTime = x.Sum(y => y.CalculatingTime), 67 TransferTime = x.Sum(y => y.TransferTime) 68 }) 69 .FirstOrDefault(); 70 return (from client in dimClientDao.GetActiveClients().Where(x => x.ResourceGroupId == id) 71 join info in factClientInfoDao.GetAll() 72 on client.Id equals info.ClientId into clientInfoJoin 73 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 74 let offline = (clientInfo.SlaveState == DA.SlaveState.Offline) 75 select new { 76 ResourceGroupId = client.ResourceGroupId, 77 GroupName = client.GroupName, 78 TotalCores = clientInfo.NumTotalCores, 79 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 80 TotalMemory = clientInfo.TotalMemory, 81 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 82 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 83 SlaveState = clientInfo.SlaveState 84 }) 85 .GroupBy(x => new { x.ResourceGroupId, x.GroupName }) 86 .Select(x => new DT.GroupDetails { 87 Id = id, 88 Name = x.Key.GroupName, 89 TotalClients = x.Count(), 90 OnlineClients = x.Count(y => y.SlaveState != DA.SlaveState.Offline), 91 TotalCores = x.Sum(y => y.TotalCores), 92 UsedCores = x.Sum(y => y.UsedCores), 93 TotalMemory = x.Sum(y => y.TotalMemory), 94 UsedMemory = x.Sum(y => y.UsedMemory), 95 TotalCpuUtilization = x.Average(y => (double?)y.CpuUtilization) ?? 0.0, 96 ActiveCpuUtilization = x.Where(y => y.SlaveState != DA.SlaveState.Offline).Average(y => (double?)y.CpuUtilization) ?? 0.0, 97 TotalUnavailableTime = clientTimeData != null ? clientTimeData.TotalUnavailableTime : 0, 98 TotalCalculatingTime = taskTimeData != null ? taskTimeData.CalculatingTime : 0, 99 TotalIdleTime = clientTimeData != null ? clientTimeData.TotalIdleTime : 0, 100 TotalOfflineTime = clientTimeData != null ? clientTimeData.TotalOfflineTime : 0, 101 TotalTransferringTime = taskTimeData != null ? taskTimeData.TransferTime : 0, 102 TasksStates = factTaskDao.GetByGroupId(id) 103 .GroupBy(y => y.TaskState) 104 .Select(y => new DT.TaskStateCount { 105 State = y.Key.ToString(), 106 Count = y.Count() 107 }).ToList() 108 }).FirstOrDefault(); 109 }); 110 } 111 112 public DT.GroupPage GetGroups(int page, int size) { 113 var pm = PersistenceManager; 114 var dimClientDao = pm.DimClientDao; 115 var factClientInfoDao = pm.FactClientInfoDao; 116 var data = (from client in dimClientDao.GetActiveClients() 71 117 join info in factClientInfoDao.GetAll() 72 118 on client.Id equals info.ClientId into clientInfoJoin … … 75 121 select new { 76 122 ResourceGroupId = client.ResourceGroupId, 77 GroupName = client.GroupName,78 123 TotalCores = clientInfo.NumTotalCores, 79 124 UsedCores = offline ? 0 : clientInfo.NumUsedCores, … … 82 127 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 83 128 SlaveState = clientInfo.SlaveState 84 }) 85 .GroupBy(x => new { x.ResourceGroupId, x.GroupName }) 86 .Select(x => new DT.GroupDetails { 87 Id = id, 88 Name = x.Key.GroupName, 89 TotalClients = x.Count(), 90 OnlineClients = x.Count(y => y.SlaveState != DA.SlaveState.Offline), 91 TotalCores = x.Sum(y => y.TotalCores), 92 UsedCores = x.Sum(y => y.UsedCores), 93 TotalMemory = x.Sum(y => y.TotalMemory), 94 UsedMemory = x.Sum(y => y.UsedMemory), 95 TotalCpuUtilization = x.Average(y => (double?)y.CpuUtilization) ?? 0.0, 96 ActiveCpuUtilization = x.Where(y => y.SlaveState != DA.SlaveState.Offline).Average(y => (double?)y.CpuUtilization) ?? 0.0, 97 TotalUnavailableTime = clientTimeData != null ? clientTimeData.TotalUnavailableTime : 0, 98 TotalCalculatingTime = taskTimeData != null ? taskTimeData.CalculatingTime : 0, 99 TotalIdleTime = clientTimeData != null ? clientTimeData.TotalIdleTime : 0, 100 TotalOfflineTime = clientTimeData != null ? clientTimeData.TotalOfflineTime : 0, 101 TotalTransferringTime = taskTimeData != null ? taskTimeData.TransferTime : 0, 102 TasksStates = factTaskDao.GetByGroupId(id) 103 .GroupBy(y => y.TaskState) 104 .Select(y => new DT.TaskStateCount { 105 State = y.Key.ToString(), 106 Count = y.Count() 107 }).ToList() 108 }).FirstOrDefault(); 129 }).GroupBy(x => x.ResourceGroupId).Select(x => new { 130 GroupId = x.Key, 131 TotalClients = x.Count(), 132 OnlineClients = x.Count(y => y.SlaveState != DA.SlaveState.Offline), 133 TotalCores = x.Sum(y => y.TotalCores), 134 UsedCores = x.Sum(y => y.UsedCores), 135 TotalMemory = x.Sum(y => y.TotalMemory), 136 UsedMemory = x.Sum(y => y.UsedMemory), 137 CpuUtilization = x.Where(y => y.SlaveState != DA.SlaveState.Offline).Average(y => (double?)y.CpuUtilization) ?? 0.0 138 }); 139 var query = dimClientDao.GetAll() 140 .GroupBy(x => new { x.ResourceGroupId, x.GroupName }) 141 .Select(x => new { 142 Id = x.Key.ResourceGroupId ?? default(Guid), 143 Name = x.Key.GroupName 109 144 }); 110 } 111 } 112 113 public DT.GroupPage GetGroups(int page, int size) { 114 using (var pm = PersistenceManager) { 115 var dimClientDao = pm.DimClientDao; 116 var factClientInfoDao = pm.FactClientInfoDao; 117 var data = (from client in dimClientDao.GetActiveClients() 118 join info in factClientInfoDao.GetAll() 119 on client.Id equals info.ClientId into clientInfoJoin 120 from clientInfo in clientInfoJoin.OrderByDescending(x => x.Time).Take(1) 121 let offline = (clientInfo.SlaveState == DA.SlaveState.Offline) 122 select new { 123 ResourceGroupId = client.ResourceGroupId, 124 TotalCores = clientInfo.NumTotalCores, 125 UsedCores = offline ? 0 : clientInfo.NumUsedCores, 126 TotalMemory = clientInfo.TotalMemory, 127 UsedMemory = offline ? 0 : clientInfo.UsedMemory, 128 CpuUtilization = offline ? 0 : clientInfo.CpuUtilization, 129 SlaveState = clientInfo.SlaveState 130 }).GroupBy(x => x.ResourceGroupId).Select(x => new { 131 GroupId = x.Key, 132 TotalClients = x.Count(), 133 OnlineClients = x.Count(y => y.SlaveState != DA.SlaveState.Offline), 134 TotalCores = x.Sum(y => y.TotalCores), 135 UsedCores = x.Sum(y => y.UsedCores), 136 TotalMemory = x.Sum(y => y.TotalMemory), 137 UsedMemory = x.Sum(y => y.UsedMemory), 138 CpuUtilization = x.Where(y => y.SlaveState != DA.SlaveState.Offline).Average(y => (double?)y.CpuUtilization) ?? 0.0 139 }); 140 var query = dimClientDao.GetAll() 141 .GroupBy(x => new { x.ResourceGroupId, x.GroupName }) 142 .Select(x => new { 143 Id = x.Key.ResourceGroupId ?? default(Guid), 144 Name = x.Key.GroupName 145 }); 146 return pm.UseTransaction(() => new DT.GroupPage { 147 TotalGroups = query.Count(), 148 Groups = query 149 .Skip((page - 1) * size) 150 .Take(size) 151 .Join(data, x => x.Id, y => y.GroupId, 152 (group, info) => new DT.Group { 153 Id = group.Id, 154 Name = group.Name, 155 TotalClients = info.TotalClients, 156 OnlineClients = info.OnlineClients, 157 TotalCores = info.TotalCores, 158 UsedCores = info.UsedCores, 159 TotalMemory = info.TotalMemory, 160 UsedMemory = info.UsedMemory, 161 CpuUtilization = info.CpuUtilization 162 }).ToList() 163 }); 164 } 145 return pm.UseTransaction(() => new DT.GroupPage { 146 TotalGroups = query.Count(), 147 Groups = query 148 .Skip((page - 1) * size) 149 .Take(size) 150 .Join(data, x => x.Id, y => y.GroupId, 151 (group, info) => new DT.Group { 152 Id = group.Id, 153 Name = group.Name, 154 TotalClients = info.TotalClients, 155 OnlineClients = info.OnlineClients, 156 TotalCores = info.TotalCores, 157 UsedCores = info.UsedCores, 158 TotalMemory = info.TotalMemory, 159 UsedMemory = info.UsedMemory, 160 CpuUtilization = info.CpuUtilization 161 }).ToList() 162 }); 165 163 } 166 164 } -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Statistics/3.3/WebApi/JobController.cs
r12562 r12858 47 47 48 48 public DT.JobDetails GetJobDetails(Guid id) { 49 using (var pm = PersistenceManager) { 50 var dimJobDao = pm.DimJobDao; 51 var factTaskDao = pm.FactTaskDao; 52 return pm.UseTransaction(() => { 53 var job = dimJobDao.GetById(id); 54 if (job != null) { 55 var timeData = factTaskDao.GetByJobId(id).GroupBy(x => x.JobId).Select(x => new { 56 AvgTransferringTime = (long)x.Average(y => y.TransferTime), 57 MinCalculatingTime = (long)x.Min(y => y.CalculatingTime), 58 MaxCalculatingTime = (long)x.Max(y => y.CalculatingTime), 59 AvgCalculatingTime = (long)x.Average(y => y.CalculatingTime), 60 TotalCalculatingTime = (long)x.Sum(y => y.CalculatingTime), 61 TotalWaitingTime = (long)x.Sum(y => y.WaitingTime) 62 }).FirstOrDefault(); 63 DateTime endTime = job.DateCompleted ?? DateTime.Now; 64 long totalTime = (long)(endTime - job.DateCreated).TotalSeconds; 65 return new DT.JobDetails { 66 Id = job.JobId, 67 Name = job.JobName, 68 UserId = job.UserId, 69 UserName = job.UserName, 70 DateCreated = job.DateCreated, 71 DateCompleted = job.DateCompleted, 72 TotalTasks = job.TotalTasks, 73 CompletedTasks = job.CompletedTasks, 74 AvgTransferringTime = timeData != null ? timeData.AvgTransferringTime : 0, 75 AvgCalculatingTime = timeData != null ? timeData.AvgCalculatingTime : 0, 76 MinCalculatingTime = timeData != null ? timeData.MinCalculatingTime : 0, 77 MaxCalculatingTime = timeData != null ? timeData.MaxCalculatingTime : 0, 78 TotalCalculatingTime = timeData != null ? timeData.TotalCalculatingTime : 0, 79 TotalWaitingTime = timeData != null ? timeData.TotalWaitingTime : 0, 80 TotalTime = totalTime, 81 TasksStates = factTaskDao.GetByJobId(id) 82 .GroupBy(x => x.TaskState) 83 .Select(x => new DT.TaskStateCount { 84 State = x.Key.ToString(), 85 Count = x.Count() 86 }).ToList() 87 }; 88 } 89 return null; 90 }); 91 } 49 var pm = PersistenceManager; 50 var dimJobDao = pm.DimJobDao; 51 var factTaskDao = pm.FactTaskDao; 52 return pm.UseTransaction(() => { 53 var job = dimJobDao.GetById(id); 54 if (job != null) { 55 var timeData = factTaskDao.GetByJobId(id).GroupBy(x => x.JobId).Select(x => new { 56 AvgTransferringTime = (long)x.Average(y => y.TransferTime), 57 MinCalculatingTime = (long)x.Min(y => y.CalculatingTime), 58 MaxCalculatingTime = (long)x.Max(y => y.CalculatingTime), 59 AvgCalculatingTime = (long)x.Average(y => y.CalculatingTime), 60 TotalCalculatingTime = (long)x.Sum(y => y.CalculatingTime), 61 TotalWaitingTime = (long)x.Sum(y => y.WaitingTime) 62 }).FirstOrDefault(); 63 DateTime endTime = job.DateCompleted ?? DateTime.Now; 64 long totalTime = (long)(endTime - job.DateCreated).TotalSeconds; 65 return new DT.JobDetails { 66 Id = job.JobId, 67 Name = job.JobName, 68 UserId = job.UserId, 69 UserName = job.UserName, 70 DateCreated = job.DateCreated, 71 DateCompleted = job.DateCompleted, 72 TotalTasks = job.TotalTasks, 73 CompletedTasks = job.CompletedTasks, 74 AvgTransferringTime = timeData != null ? timeData.AvgTransferringTime : 0, 75 AvgCalculatingTime = timeData != null ? timeData.AvgCalculatingTime : 0, 76 MinCalculatingTime = timeData != null ? timeData.MinCalculatingTime : 0, 77 MaxCalculatingTime = timeData != null ? timeData.MaxCalculatingTime : 0, 78 TotalCalculatingTime = timeData != null ? timeData.TotalCalculatingTime : 0, 79 TotalWaitingTime = timeData != null ? timeData.TotalWaitingTime : 0, 80 TotalTime = totalTime, 81 TasksStates = factTaskDao.GetByJobId(id) 82 .GroupBy(x => x.TaskState) 83 .Select(x => new DT.TaskStateCount { 84 State = x.Key.ToString(), 85 Count = x.Count() 86 }).ToList() 87 }; 88 } 89 return null; 90 }); 92 91 } 93 92 … … 100 99 RoleVerifier.AuthenticateForAnyRole(HiveRoles.Administrator); 101 100 } 102 using (var pm = PersistenceManager) { 103 var dimJobDao = pm.DimJobDao; 104 return pm.UseTransaction(() => { 105 var jobs = dimJobDao.GetByUserId(id).Where(x => completed ? (x.DateCompleted != null) : (x.DateCompleted == null)); 106 return new DT.JobPage { 107 TotalJobs = jobs.Count(), 108 Jobs = jobs.OrderByDescending(x => x.DateCompleted ?? DateTime.MaxValue) 109 .Skip((page - 1) * size) 110 .Take(size) 111 .Select(x => ConvertToDT(x)) 112 .ToList() 113 }; 114 }); 115 } 101 var pm = PersistenceManager; 102 var dimJobDao = pm.DimJobDao; 103 return pm.UseTransaction(() => { 104 var jobs = dimJobDao.GetByUserId(id).Where(x => completed ? (x.DateCompleted != null) : (x.DateCompleted == null)); 105 return new DT.JobPage { 106 TotalJobs = jobs.Count(), 107 Jobs = jobs.OrderByDescending(x => x.DateCompleted ?? DateTime.MaxValue) 108 .Skip((page - 1) * size) 109 .Take(size) 110 .Select(x => ConvertToDT(x)) 111 .ToList() 112 }; 113 }); 116 114 } 117 115 … … 124 122 RoleVerifier.AuthenticateForAnyRole(HiveRoles.Administrator); 125 123 } 126 using (var pm = PersistenceManager) { 127 var dimJobDao = pm.DimJobDao; 128 return pm.UseTransaction(() => { 129 return dimJobDao.GetByUserId(id) 130 .Where(x => completed ? (x.DateCompleted != null) : (x.DateCompleted == null)) 131 .Select(x => ConvertToDT(x)) 132 .ToList(); 133 }); 134 } 124 var pm = PersistenceManager; 125 var dimJobDao = pm.DimJobDao; 126 return pm.UseTransaction(() => { 127 return dimJobDao.GetByUserId(id) 128 .Where(x => completed ? (x.DateCompleted != null) : (x.DateCompleted == null)) 129 .Select(x => ConvertToDT(x)) 130 .ToList(); 131 }); 135 132 } 136 133 137 134 [Authorize(Roles = HiveRoles.Administrator)] 138 135 public IEnumerable<DT.Job> GetAllActiveJobsFromAllUsers() { 139 using (var pm = PersistenceManager) { 140 var dimJobDao = pm.DimJobDao; 141 return pm.UseTransaction(() => { 142 return dimJobDao.GetAll() 143 .Where(x => x.DateCompleted == null) 144 .OrderByDescending(x => x.DateCreated) 145 .Select(x => ConvertToDT(x)) 146 .ToList(); 147 }); 148 } 136 var pm = PersistenceManager; 137 var dimJobDao = pm.DimJobDao; 138 return pm.UseTransaction(() => { 139 return dimJobDao.GetAll() 140 .Where(x => x.DateCompleted == null) 141 .OrderByDescending(x => x.DateCreated) 142 .Select(x => ConvertToDT(x)) 143 .ToList(); 144 }); 149 145 } 150 146 -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Statistics/3.3/WebApi/TaskController.cs
r12768 r12858 51 51 [HttpPost] 52 52 public DT.TaskPage GetTasksByJobId(Guid id, int page, int size, IEnumerable<string> states) { 53 using (var pm = PersistenceManager) {54 55 56 53 var pm = PersistenceManager; 54 var dimJobDao = pm.DimJobDao; 55 var dimClientDao = pm.DimClientDao; 56 var factTaskDao = pm.FactTaskDao; 57 57 58 59 60 61 62 63 64 58 DA.DimJob job = pm.UseTransaction(() => dimJobDao.GetById(id)); 59 if (job == null) { 60 throw new ArgumentException("invalid job id"); 61 } 62 if (job.UserId != UserManager.CurrentUserId) { 63 RoleVerifier.AuthenticateForAnyRole(HiveRoles.Administrator); 64 } 65 65 66 return pm.UseTransaction(() => { 67 var tasks = factTaskDao.GetByJobId(id).Where(x => states.Contains(x.TaskState.ToString())); 68 return new DT.TaskPage { 69 TotalTasks = tasks.Count(), 70 Tasks = (from factTask in tasks 71 join dimJob in dimJobDao.GetAll() on factTask.JobId equals dimJob.JobId 72 join dimClient in dimClientDao.GetAll() on factTask.LastClientId equals 73 dimClient.Id into taskClientJoin 74 from a in taskClientJoin.DefaultIfEmpty() 75 let startTime = factTask.StartTime ?? DateTime.Now 76 let endTime = factTask.EndTime ?? DateTime.Now 77 select new DT.Task { 78 Id = factTask.TaskId, 79 JobId = factTask.JobId, 80 JobName = dimJob.JobName, 81 TotalTime = (long)(endTime - startTime).TotalSeconds, 82 CalculatingTime = factTask.CalculatingTime, 83 WaitingTime = factTask.WaitingTime, 84 TransferTime = factTask.TransferTime, 85 InitialWaitingTime = factTask.InitialWaitingTime, 86 NumCalculationRuns = factTask.NumCalculationRuns, 87 NumRetries = factTask.NumRetries, 88 CoresRequired = factTask.CoresRequired, 89 MemoryRequired = factTask.MemoryRequired, 90 Priority = factTask.Priority, 91 State = factTask.TaskState.ToString(), 92 LastClientId = factTask.LastClientId, 93 LastClientName = a != null ? a.Name : string.Empty, 94 UserId = dimJob.UserId, 95 UserName = dimJob.UserName, 96 StartTime = factTask.StartTime, 97 EndTime = factTask.EndTime, 98 Exception = factTask.Exception 99 }) 100 .Skip((page - 1) * size) 101 .Take(size) 102 .ToList() 103 }; 104 }); 105 } 66 return pm.UseTransaction(() => { 67 var tasks = factTaskDao.GetByJobId(id).Where(x => states.Contains(x.TaskState.ToString())); 68 return new DT.TaskPage { 69 TotalTasks = tasks.Count(), 70 Tasks = (from factTask in tasks 71 join dimJob in dimJobDao.GetAll() on factTask.JobId equals dimJob.JobId 72 join dimClient in dimClientDao.GetAll() on factTask.LastClientId equals 73 dimClient.Id into taskClientJoin 74 from a in taskClientJoin.DefaultIfEmpty() 75 let startTime = factTask.StartTime ?? DateTime.Now 76 let endTime = factTask.EndTime ?? DateTime.Now 77 select new DT.Task { 78 Id = factTask.TaskId, 79 JobId = factTask.JobId, 80 JobName = dimJob.JobName, 81 TotalTime = (long)(endTime - startTime).TotalSeconds, 82 CalculatingTime = factTask.CalculatingTime, 83 WaitingTime = factTask.WaitingTime, 84 TransferTime = factTask.TransferTime, 85 InitialWaitingTime = factTask.InitialWaitingTime, 86 NumCalculationRuns = factTask.NumCalculationRuns, 87 NumRetries = factTask.NumRetries, 88 CoresRequired = factTask.CoresRequired, 89 MemoryRequired = factTask.MemoryRequired, 90 Priority = factTask.Priority, 91 State = factTask.TaskState.ToString(), 92 LastClientId = factTask.LastClientId, 93 LastClientName = a != null ? a.Name : string.Empty, 94 UserId = dimJob.UserId, 95 UserName = dimJob.UserName, 96 StartTime = factTask.StartTime, 97 EndTime = factTask.EndTime, 98 Exception = factTask.Exception 99 }) 100 .Skip((page - 1) * size) 101 .Take(size) 102 .ToList() 103 }; 104 }); 106 105 } 107 106 … … 109 108 public DT.TaskPage GetTasksByClientId(Guid id, int page, int size, IEnumerable<string> states, Guid userId = default(Guid)) { 110 109 bool isAdministrator = User.IsInRole(HiveRoles.Administrator); 111 using (var pm = PersistenceManager) { 112 var dimJobDao = pm.DimJobDao; 113 var dimClientDao = pm.DimClientDao; 114 var factTaskDao = pm.FactTaskDao; 115 return pm.UseTransaction(() => { 116 var tasks = factTaskDao.GetByClientId(id).Where(x => states.Contains(x.TaskState.ToString())); 117 if (userId != Guid.Empty) { 118 tasks = tasks.Where(x => x.DimJob.UserId == userId); 119 } 120 return new DT.TaskPage { 121 TotalTasks = tasks.Count(), 122 Tasks = (from factTask in tasks 123 join dimJob in dimJobDao.GetAll() on factTask.JobId equals dimJob.JobId 124 join dimClient in dimClientDao.GetAll() on factTask.LastClientId equals 125 dimClient.Id into taskClientJoin 126 from a in taskClientJoin.DefaultIfEmpty() 127 let startTime = factTask.StartTime ?? DateTime.Now 128 let endTime = factTask.EndTime ?? DateTime.Now 129 select new DT.Task { 130 Id = isAdministrator ? factTask.TaskId : default(Guid), 131 JobId = isAdministrator ? factTask.JobId : default(Guid), 132 JobName = isAdministrator ? dimJob.JobName : string.Empty, 133 TotalTime = (long)(endTime - startTime).TotalSeconds, 134 CalculatingTime = factTask.CalculatingTime, 135 WaitingTime = factTask.WaitingTime, 136 TransferTime = factTask.TransferTime, 137 InitialWaitingTime = factTask.InitialWaitingTime, 138 NumCalculationRuns = factTask.NumCalculationRuns, 139 NumRetries = factTask.NumRetries, 140 CoresRequired = factTask.CoresRequired, 141 MemoryRequired = factTask.MemoryRequired, 142 Priority = factTask.Priority, 143 State = factTask.TaskState.ToString(), 144 LastClientId = factTask.LastClientId, 145 LastClientName = a != null ? a.Name : string.Empty, 146 UserId = isAdministrator ? dimJob.UserId : default(Guid), 147 UserName = isAdministrator ? dimJob.UserName : string.Empty, 148 StartTime = factTask.StartTime, 149 EndTime = factTask.EndTime, 150 Exception = isAdministrator ? factTask.Exception : string.Empty 151 }) 152 .OrderByDescending(x => x.EndTime ?? DateTime.MaxValue) 153 .Skip((page - 1) * size) 154 .Take(size) 155 .ToList() 156 }; 157 }); 158 } 110 var pm = PersistenceManager; 111 var dimJobDao = pm.DimJobDao; 112 var dimClientDao = pm.DimClientDao; 113 var factTaskDao = pm.FactTaskDao; 114 return pm.UseTransaction(() => { 115 var tasks = factTaskDao.GetByClientId(id).Where(x => states.Contains(x.TaskState.ToString())); 116 if (userId != Guid.Empty) { 117 tasks = tasks.Where(x => x.DimJob.UserId == userId); 118 } 119 return new DT.TaskPage { 120 TotalTasks = tasks.Count(), 121 Tasks = (from factTask in tasks 122 join dimJob in dimJobDao.GetAll() on factTask.JobId equals dimJob.JobId 123 join dimClient in dimClientDao.GetAll() on factTask.LastClientId equals 124 dimClient.Id into taskClientJoin 125 from a in taskClientJoin.DefaultIfEmpty() 126 let startTime = factTask.StartTime ?? DateTime.Now 127 let endTime = factTask.EndTime ?? DateTime.Now 128 select new DT.Task { 129 Id = isAdministrator ? factTask.TaskId : default(Guid), 130 JobId = isAdministrator ? factTask.JobId : default(Guid), 131 JobName = isAdministrator ? dimJob.JobName : string.Empty, 132 TotalTime = (long)(endTime - startTime).TotalSeconds, 133 CalculatingTime = factTask.CalculatingTime, 134 WaitingTime = factTask.WaitingTime, 135 TransferTime = factTask.TransferTime, 136 InitialWaitingTime = factTask.InitialWaitingTime, 137 NumCalculationRuns = factTask.NumCalculationRuns, 138 NumRetries = factTask.NumRetries, 139 CoresRequired = factTask.CoresRequired, 140 MemoryRequired = factTask.MemoryRequired, 141 Priority = factTask.Priority, 142 State = factTask.TaskState.ToString(), 143 LastClientId = factTask.LastClientId, 144 LastClientName = a != null ? a.Name : string.Empty, 145 UserId = isAdministrator ? dimJob.UserId : default(Guid), 146 UserName = isAdministrator ? dimJob.UserName : string.Empty, 147 StartTime = factTask.StartTime, 148 EndTime = factTask.EndTime, 149 Exception = isAdministrator ? factTask.Exception : string.Empty 150 }) 151 .OrderByDescending(x => x.EndTime ?? DateTime.MaxValue) 152 .Skip((page - 1) * size) 153 .Take(size) 154 .ToList() 155 }; 156 }); 159 157 } 160 158 161 159 public HttpResponseMessage GetTaskDataById(Guid id) { 162 using (var pm = PersistenceManager) { 163 var taskDataDao = pm.TaskDataDao; 164 return pm.UseTransaction(() => { 165 var taskData = taskDataDao.GetById(id); 166 if (taskData == null) 167 return new HttpResponseMessage(HttpStatusCode.NotFound); 168 HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK); 169 var stream = new MemoryStream(taskData.Data); 170 result.Content = new StreamContent(stream); 171 result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); 172 result.Content.Headers.ContentDisposition = 173 new ContentDispositionHeaderValue("attachment") { 174 FileName = string.Format("{0}.hl", id) 175 }; 176 return result; 177 }); 178 } 160 var pm = PersistenceManager; 161 var taskDataDao = pm.TaskDataDao; 162 return pm.UseTransaction(() => { 163 var taskData = taskDataDao.GetById(id); 164 if (taskData == null) 165 return new HttpResponseMessage(HttpStatusCode.NotFound); 166 HttpResponseMessage result = new HttpResponseMessage(HttpStatusCode.OK); 167 var stream = new MemoryStream(taskData.Data); 168 result.Content = new StreamContent(stream); 169 result.Content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); 170 result.Content.Headers.ContentDisposition = 171 new ContentDispositionHeaderValue("attachment") { 172 FileName = string.Format("{0}.hl", id) 173 }; 174 return result; 175 }); 179 176 } 180 177 } -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Statistics/3.3/WebApi/UserController.cs
r12562 r12858 37 37 38 38 public IEnumerable<DT.User> GetUsers() { 39 using (var pm = PersistenceManager) { 40 var dimUserDao = pm.DimUserDao; 41 return pm.UseTransaction(() => { 42 return dimUserDao.GetAll().Select(x => new DT.User { 43 Id = x.UserId, 44 Name = x.Name 45 }).ToList(); 46 }); 47 } 39 var pm = PersistenceManager; 40 var dimUserDao = pm.DimUserDao; 41 return pm.UseTransaction(() => { 42 return dimUserDao.GetAll().Select(x => new DT.User { 43 Id = x.UserId, 44 Name = x.Name 45 }).ToList(); 46 }); 48 47 } 49 48 50 49 public DT.UserDetails GetUser(Guid id) { 51 using (var pm = PersistenceManager) { 52 var dimUserDao = pm.DimUserDao; 53 var factTaskDao = pm.FactTaskDao; 54 return pm.UseTransaction(() => { 55 var user = dimUserDao.GetById(id); 56 if (user != null) { 57 return new DT.UserDetails { 58 Id = user.UserId, 59 Name = user.Name, 60 TasksStates = factTaskDao.GetByUserId(id) 61 .GroupBy(x => x.TaskState) 62 .Select(x => new DT.TaskStateCount { 63 State = x.Key.ToString(), 64 Count = x.Count() 65 }).ToList() 66 }; 67 } 68 return null; 69 }); 70 } 50 var pm = PersistenceManager; 51 var dimUserDao = pm.DimUserDao; 52 var factTaskDao = pm.FactTaskDao; 53 return pm.UseTransaction(() => { 54 var user = dimUserDao.GetById(id); 55 if (user != null) { 56 return new DT.UserDetails { 57 Id = user.UserId, 58 Name = user.Name, 59 TasksStates = factTaskDao.GetByUserId(id) 60 .GroupBy(x => x.TaskState) 61 .Select(x => new DT.TaskStateCount { 62 State = x.Key.ToString(), 63 Count = x.Count() 64 }).ToList() 65 }; 66 } 67 return null; 68 }); 71 69 } 72 70 } -
branches/HiveStatistics/sources/HeuristicLab.Services.WebApp.Status/3.3/WebApi/DataController.cs
r12691 r12858 99 99 100 100 public DT.Status GetStatus() { 101 using (var pm = PersistenceManager) { 102 var slaveDao = pm.SlaveDao; 103 var onlineSlaves = pm.UseTransaction(() => slaveDao.GetOnlineSlaves().ToList()); 104 var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList(); 105 var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList(); 106 int totalCores = onlineSlaves.Sum(s => s.Cores ?? 0); 107 int totalMemory = onlineSlaves.Sum(s => s.Memory ?? 0); 108 return new DT.Status { 109 CoreStatus = new DT.CoreStatus { 110 TotalCores = totalCores, 111 UsedCores = totalCores - onlineSlaves.Sum(s => s.FreeCores ?? 0), 112 ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0), 113 CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0) - calculatingSlaves.Sum(s => s.FreeCores ?? 0) 101 var pm = PersistenceManager; 102 var slaveDao = pm.SlaveDao; 103 var onlineSlaves = pm.UseTransaction(() => slaveDao.GetOnlineSlaves().ToList()); 104 var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList(); 105 var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList(); 106 int totalCores = onlineSlaves.Sum(s => s.Cores ?? 0); 107 int totalMemory = onlineSlaves.Sum(s => s.Memory ?? 0); 108 return new DT.Status { 109 CoreStatus = new DT.CoreStatus { 110 TotalCores = totalCores, 111 UsedCores = totalCores - onlineSlaves.Sum(s => s.FreeCores ?? 0), 112 ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0), 113 CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0) - calculatingSlaves.Sum(s => s.FreeCores ?? 0) 114 }, 115 CpuUtilizationStatus = new DT.CpuUtilizationStatus { 116 TotalCpuUtilization = onlineSlaves.Any() 117 ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2) 118 : 0.0, 119 ActiveCpuUtilization = activeSlaves.Any() 120 ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2) 121 : 0.0, 122 CalculatingCpuUtilization = calculatingSlaves.Any() 123 ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2) 124 : 0.0 125 }, 126 MemoryStatus = new DT.MemoryStatus { 127 TotalMemory = totalMemory, 128 UsedMemory = totalMemory - onlineSlaves.Sum(s => s.FreeMemory ?? 0), 129 ActiveMemory = activeSlaves.Sum(s => s.Memory ?? 0), 130 CalculatingMemory = calculatingSlaves.Sum(s => s.Memory ?? 0) - calculatingSlaves.Sum(s => s.FreeMemory ?? 0) 131 }, 132 TimeStatus = GetTimeStatus(pm), 133 TasksStatus = GetTaskStatus(pm), 134 SlavesStatus = onlineSlaves.Select(x => new DT.SlaveStatus { 135 Slave = new DT.Slave { 136 Id = x.ResourceId.ToString(), 137 Name = x.Name 114 138 }, 115 CpuUtilizationStatus = new DT.CpuUtilizationStatus { 116 TotalCpuUtilization = onlineSlaves.Any() 117 ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2) 118 : 0.0, 119 ActiveCpuUtilization = activeSlaves.Any() 120 ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2) 121 : 0.0, 122 CalculatingCpuUtilization = calculatingSlaves.Any() 123 ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2) 124 : 0.0 125 }, 126 MemoryStatus = new DT.MemoryStatus { 127 TotalMemory = totalMemory, 128 UsedMemory = totalMemory - onlineSlaves.Sum(s => s.FreeMemory ?? 0), 129 ActiveMemory = activeSlaves.Sum(s => s.Memory ?? 0), 130 CalculatingMemory = calculatingSlaves.Sum(s => s.Memory ?? 0) - calculatingSlaves.Sum(s => s.FreeMemory ?? 0) 131 }, 132 TimeStatus = GetTimeStatus(pm), 133 TasksStatus = GetTaskStatus(pm), 134 SlavesStatus = onlineSlaves.Select(x => new DT.SlaveStatus { 135 Slave = new DT.Slave { 136 Id = x.ResourceId.ToString(), 137 Name = x.Name 138 }, 139 CpuUtilization = x.CpuUtilization, 140 Cores = x.Cores ?? 0, 141 FreeCores = x.FreeCores ?? 0, 142 Memory = x.Memory ?? 0, 143 FreeMemory = x.FreeMemory ?? 0, 144 IsAllowedToCalculate = x.IsAllowedToCalculate, 145 State = x.SlaveState.ToString() 146 }).OrderBy(x => x.Slave.Name), 147 Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now) 148 }; 149 } 139 CpuUtilization = x.CpuUtilization, 140 Cores = x.Cores ?? 0, 141 FreeCores = x.FreeCores ?? 0, 142 Memory = x.Memory ?? 0, 143 FreeMemory = x.FreeMemory ?? 0, 144 IsAllowedToCalculate = x.IsAllowedToCalculate, 145 State = x.SlaveState.ToString() 146 }).OrderBy(x => x.Slave.Name), 147 Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now) 148 }; 150 149 } 151 150 … … 158 157 increment += 5; 159 158 } 160 using (var pm = PersistenceManager) { 161 var factClientInfoDao = pm.FactClientInfoDao; 162 var clientInfos = pm.UseTransaction(() => { 163 return factClientInfoDao.GetAll() 164 .Where(s => s.Time >= start 165 && s.Time <= end 166 && s.SlaveState != SlaveState.Offline) 167 .OrderBy(s => s.Time) 168 .GroupBy(s => s.Time) 169 .Select(x => new { 170 Timestamp = x.Key, 171 TotalCores = x.Sum(y => y.NumTotalCores), 172 UsedCores = x.Sum(y => y.NumUsedCores), 173 TotalMemory = x.Sum(y => y.TotalMemory), 174 UsedMemory = x.Sum(y => y.UsedMemory), 175 CpuUtilization = x.Where(y => y.IsAllowedToCalculate).Average(y => y.CpuUtilization) 176 }) 177 .ToList(); 178 }); 179 var statusList = new List<DT.Status>(); 180 var e = clientInfos.GetEnumerator(); 181 do { 182 var status = new DT.Status { 183 CoreStatus = new DT.CoreStatus(), 184 CpuUtilizationStatus = new DT.CpuUtilizationStatus(), 185 MemoryStatus = new DT.MemoryStatus() 186 }; 187 int i = 0; 188 DateTime lastTimestamp = DateTime.Now; 189 while (e.MoveNext()) { 190 var clientInfo = e.Current; 191 status.CoreStatus.TotalCores += clientInfo.TotalCores; 192 status.CoreStatus.UsedCores += clientInfo.UsedCores; 193 status.MemoryStatus.TotalMemory += clientInfo.TotalMemory; 194 status.MemoryStatus.UsedMemory += clientInfo.UsedMemory; 195 status.CpuUtilizationStatus.TotalCpuUtilization += clientInfo.CpuUtilization; 196 lastTimestamp = clientInfo.Timestamp; 197 i++; 198 if (i >= increment) 199 break; 200 } 201 if (i <= 0) continue; 202 status.Timestamp = JavascriptUtils.ToTimestamp(lastTimestamp); 203 status.CoreStatus.TotalCores /= i; 204 status.CoreStatus.UsedCores /= i; 205 status.MemoryStatus.TotalMemory /= i; 206 status.MemoryStatus.UsedMemory /= i; 207 status.CpuUtilizationStatus.TotalCpuUtilization /= i; 208 statusList.Add(status); 209 } while (e.Current != null); 210 return statusList; 211 } 159 var pm = PersistenceManager; 160 var factClientInfoDao = pm.FactClientInfoDao; 161 var clientInfos = pm.UseTransaction(() => { 162 return factClientInfoDao.GetAll() 163 .Where(s => s.Time >= start 164 && s.Time <= end 165 && s.SlaveState != SlaveState.Offline) 166 .OrderBy(s => s.Time) 167 .GroupBy(s => s.Time) 168 .Select(x => new { 169 Timestamp = x.Key, 170 TotalCores = x.Sum(y => y.NumTotalCores), 171 UsedCores = x.Sum(y => y.NumUsedCores), 172 TotalMemory = x.Sum(y => y.TotalMemory), 173 UsedMemory = x.Sum(y => y.UsedMemory), 174 CpuUtilization = x.Where(y => y.IsAllowedToCalculate).Average(y => y.CpuUtilization) 175 }) 176 .ToList(); 177 }); 178 var statusList = new List<DT.Status>(); 179 var e = clientInfos.GetEnumerator(); 180 do { 181 var status = new DT.Status { 182 CoreStatus = new DT.CoreStatus(), 183 CpuUtilizationStatus = new DT.CpuUtilizationStatus(), 184 MemoryStatus = new DT.MemoryStatus() 185 }; 186 int i = 0; 187 DateTime lastTimestamp = DateTime.Now; 188 while (e.MoveNext()) { 189 var clientInfo = e.Current; 190 status.CoreStatus.TotalCores += clientInfo.TotalCores; 191 status.CoreStatus.UsedCores += clientInfo.UsedCores; 192 status.MemoryStatus.TotalMemory += clientInfo.TotalMemory; 193 status.MemoryStatus.UsedMemory += clientInfo.UsedMemory; 194 status.CpuUtilizationStatus.TotalCpuUtilization += clientInfo.CpuUtilization; 195 lastTimestamp = clientInfo.Timestamp; 196 i++; 197 if (i >= increment) 198 break; 199 } 200 if (i <= 0) continue; 201 status.Timestamp = JavascriptUtils.ToTimestamp(lastTimestamp); 202 status.CoreStatus.TotalCores /= i; 203 status.CoreStatus.UsedCores /= i; 204 status.MemoryStatus.TotalMemory /= i; 205 status.MemoryStatus.UsedMemory /= i; 206 status.CpuUtilizationStatus.TotalCpuUtilization /= i; 207 statusList.Add(status); 208 } while (e.Current != null); 209 return statusList; 212 210 } 213 211
Note: See TracChangeset
for help on using the changeset viewer.