 Timestamp:
 09/21/15 16:28:38 (7 years ago)
 Location:
 stable
 Files:

 2 edited
Legend:
 Unmodified
 Added
 Removed

stable
 Property svn:mergeinfo changed

stable/HeuristicLab.Services.WebApp.Status/3.3/WebApi/DataController.cs
r12563 r12962 22 22 using System; 23 23 using System.Collections.Generic; 24 using System.Data.Linq;25 24 using System.Linq; 26 25 using System.Web.Http; 27 26 using HeuristicLab.Services.Hive; 28 27 using HeuristicLab.Services.Hive.DataAccess; 29 using DAL = HeuristicLab.Services.Hive.DataAccess;30 using DT O= HeuristicLab.Services.WebApp.Status.WebApi.DataTransfer;28 using HeuristicLab.Services.Hive.DataAccess.Interfaces; 29 using DT = HeuristicLab.Services.WebApp.Status.WebApi.DataTransfer; 31 30 32 31 namespace HeuristicLab.Services.WebApp.Status.WebApi { 33 32 public class DataController : ApiController { 34 35 // start temporary quickfix 33 private const int LAST_TASKS = 20; 34 35 private IPersistenceManager PersistenceManager { 36 get { return ServiceLocator.Instance.PersistenceManager; } 37 } 38 36 39 private const string SQL_USER_TASK_STATUS = 37 40 @"WITH UserTasks AS ( … … 53 56 } 54 57 55 public IEnumerable<DTO.TaskStatus> GetTaskStatus(HiveDataContext db) { 56 var query = db.ExecuteQuery<UserTaskStatus>(SQL_USER_TASK_STATUS).ToList(); 57 return query.Select(uts => new DTO.TaskStatus { 58 User = new DTO.User { 59 Id = uts.UserId.ToString(), 60 Name = ServiceLocator.Instance.UserManager.GetUserById(uts.UserId).UserName 58 private IEnumerable<DT.TaskStatus> GetTaskStatus(IPersistenceManager pm) { 59 return pm.UseTransaction(() => { 60 var query = pm.DataContext.ExecuteQuery<UserTaskStatus>(SQL_USER_TASK_STATUS).ToList(); 61 return query.Select(uts => new DT.TaskStatus { 62 User = new DT.User { 63 Id = uts.UserId.ToString(), 64 Name = ServiceLocator.Instance.UserManager.GetUserNameById(uts.UserId) 65 }, 66 CalculatingTasks = uts.CalculatingTasks, 67 WaitingTasks = uts.WaitingTasks 68 }).OrderBy(x => x.User.Name); 69 }); 70 } 71 72 private DT.TimeStatus GetTimeStatus(IPersistenceManager pm) { 73 return pm.UseTransaction(() => { 74 var factTaskDao = pm.FactTaskDao; 75 var factTasks = factTaskDao.GetAll(); 76 var completedTasks = factTaskDao.GetCompletedTasks() 77 .OrderByDescending(x => x.EndTime) 78 .Take(LAST_TASKS); 79 var lastCalculatingTimes = completedTasks 80 .GroupBy(x => 1) 81 .Select(x => new { 82 Min = x.Min(y => y.CalculatingTime), 83 Max = x.Max(y => y.CalculatingTime), 84 Avg = (long)x.Average(y => (long?)y.CalculatingTime) 85 }).FirstOrDefault(); 86 var calculatingTasks = factTasks.Where(x => x.TaskState == TaskState.Calculating); 87 int count = calculatingTasks.Count() / 3; 88 return new DT.TimeStatus { 89 MinCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Min : 0, 90 MaxCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Max : 0, 91 AvgWaitingTime = count > 0 ? (long)calculatingTasks.OrderBy(x => x.StartTime).Take(count).Average(x => x.InitialWaitingTime) : 0, 92 AvgCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Avg : 0, 93 TotalCpuTime = factTasks.Sum(x => Convert.ToInt64(x.CalculatingTime)), 94 StandardDeviationCalculatingTime = (long)StandardDeviation(completedTasks.Select(x => (double)x.CalculatingTime)), 95 BeginDate = factTasks.Where(x => x.StartTime.HasValue).OrderBy(x => x.StartTime).Select(x => x.StartTime).FirstOrDefault() 96 }; 97 }); 98 } 99 100 public DT.Status GetStatus() { 101 var pm = PersistenceManager; 102 var slaveDao = pm.SlaveDao; 103 var onlineSlaves = pm.UseTransaction(() => slaveDao.GetOnlineSlaves().ToList()); 104 var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList(); 105 var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList(); 106 int totalCores = onlineSlaves.Sum(s => s.Cores ?? 0); 107 int totalMemory = onlineSlaves.Sum(s => s.Memory ?? 0); 108 return new DT.Status { 109 CoreStatus = new DT.CoreStatus { 110 TotalCores = totalCores, 111 UsedCores = totalCores  onlineSlaves.Sum(s => s.FreeCores ?? 0), 112 ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0), 113 CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0)  calculatingSlaves.Sum(s => s.FreeCores ?? 0) 61 114 }, 62 CalculatingTasks = uts.CalculatingTasks, 63 WaitingTasks = uts.WaitingTasks 64 }).OrderBy(x => x.User.Name); 65 } 66 // end temporary quickfix 67 68 public DTO.Status GetStatus() { 69 using (var db = new HiveDataContext()) { 70 var onlineSlaves = (from slave in db.Resources.OfType<DAL.Slave>() 71 where slave.SlaveState == SlaveState.Calculating  slave.SlaveState == SlaveState.Idle 72 select slave).ToList(); 73 var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList(); 74 var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList(); 75 int calculatingMemory = calculatingSlaves.Any() ? (int)calculatingSlaves.Sum(s => s.Memory) : 0; 76 int freeCalculatingMemory = calculatingSlaves.Any() ? (int)calculatingSlaves.Sum(s => s.FreeMemory) : 0; 77 78 return new DTO.Status { 79 CoreStatus = new DTO.CoreStatus { 80 TotalCores = onlineSlaves.Sum(s => s.Cores ?? 0), 81 FreeCores = onlineSlaves.Sum(s => s.FreeCores ?? 0), // temporary for old chart data 82 ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0), 83 CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0)  calculatingSlaves.Sum(s => s.FreeCores ?? 0) 115 CpuUtilizationStatus = new DT.CpuUtilizationStatus { 116 TotalCpuUtilization = onlineSlaves.Any() 117 ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2) 118 : 0.0, 119 ActiveCpuUtilization = activeSlaves.Any() 120 ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2) 121 : 0.0, 122 CalculatingCpuUtilization = calculatingSlaves.Any() 123 ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2) 124 : 0.0 125 }, 126 MemoryStatus = new DT.MemoryStatus { 127 TotalMemory = totalMemory, 128 UsedMemory = totalMemory  onlineSlaves.Sum(s => s.FreeMemory ?? 0), 129 ActiveMemory = activeSlaves.Sum(s => s.Memory ?? 0), 130 CalculatingMemory = calculatingSlaves.Sum(s => s.Memory ?? 0)  calculatingSlaves.Sum(s => s.FreeMemory ?? 0) 131 }, 132 TimeStatus = GetTimeStatus(pm), 133 TasksStatus = GetTaskStatus(pm), 134 SlavesStatus = onlineSlaves.Select(x => new DT.SlaveStatus { 135 Slave = new DT.Slave { 136 Id = x.ResourceId.ToString(), 137 Name = x.Name 84 138 }, 85 CpuUtilizationStatus = new DTO.CpuUtilizationStatus { 86 TotalCpuUtilization = onlineSlaves.Any() 87 ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2) 88 : 0.0, 89 ActiveCpuUtilization = activeSlaves.Any() 90 ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2) 91 : 0.0, 92 CalculatingCpuUtilization = calculatingSlaves.Any() 93 ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2) 94 : 0.0 95 }, 96 MemoryStatus = new DTO.MemoryStatus { 97 TotalMemory = onlineSlaves.Any() ? (int)onlineSlaves.Sum(s => s.Memory) : 0, 98 FreeMemory = onlineSlaves.Any() ? (int)onlineSlaves.Sum(s => s.FreeMemory) : 0, 99 ActiveMemory = activeSlaves.Any() ? (int)activeSlaves.Sum(s => s.Memory) : 0, 100 UsedMemory = calculatingMemory  freeCalculatingMemory 101 }, 102 TasksStatus = GetTaskStatus(db), 103 SlavesStatus = onlineSlaves.Select(x => new DTO.SlaveStatus { 104 Slave = new DTO.Slave { 105 Id = x.ResourceId.ToString(), 106 Name = x.Name 107 }, 108 CpuUtilization = x.CpuUtilization, 109 Cores = x.Cores ?? 0, 110 FreeCores = x.FreeCores ?? 0, 111 Memory = x.Memory ?? 0, 112 FreeMemory = x.FreeMemory ?? 0, 113 IsAllowedToCalculate = x.IsAllowedToCalculate, 114 State = x.SlaveState.ToString() 115 }).OrderBy(x => x.Slave.Name), 116 Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now) 117 }; 118 } 119 } 120 121 public IEnumerable<DTO.Status> GetStatusHistory(DateTime start, DateTime end) { 139 CpuUtilization = x.CpuUtilization, 140 Cores = x.Cores ?? 0, 141 FreeCores = x.FreeCores ?? 0, 142 Memory = x.Memory ?? 0, 143 FreeMemory = x.FreeMemory ?? 0, 144 IsAllowedToCalculate = x.IsAllowedToCalculate, 145 State = x.SlaveState.ToString() 146 }).OrderBy(x => x.Slave.Name), 147 Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now) 148 }; 149 } 150 151 public IEnumerable<DT.Status> GetStatusHistory(DateTime start, DateTime end) { 122 152 TimeSpan ts = end  start; 123 153 int increment = 1; … … 127 157 increment += 5; 128 158 } 129 using (var db = new HiveDataContext()) { 130 DataLoadOptions loadOptions = new DataLoadOptions(); 131 loadOptions.LoadWith<Statistics>(o => o.SlaveStatistics); 132 db.LoadOptions = loadOptions; 133 db.DeferredLoadingEnabled = false; 134 var statistics = db.Statistics.Where(s => s.Timestamp >= start && s.Timestamp <= end) 135 .OrderBy(s => s.Timestamp) 136 .ToList(); 137 var status = new DTO.Status { 138 CoreStatus = new DTO.CoreStatus(), 139 CpuUtilizationStatus = new DTO.CpuUtilizationStatus(), 140 MemoryStatus = new DTO.MemoryStatus() 159 var pm = PersistenceManager; 160 var factClientInfoDao = pm.FactClientInfoDao; 161 var clientInfos = pm.UseTransaction(() => { 162 return factClientInfoDao.GetAll() 163 .Where(s => s.Time >= start 164 && s.Time <= end 165 && s.SlaveState != SlaveState.Offline) 166 .GroupBy(s => s.Time) 167 .Select(x => new { 168 Timestamp = x.Key, 169 TotalCores = x.Sum(y => y.NumTotalCores), 170 UsedCores = x.Sum(y => y.NumUsedCores), 171 TotalMemory = x.Sum(y => y.TotalMemory), 172 UsedMemory = x.Sum(y => y.UsedMemory), 173 CpuUtilization = x.Where(y => y.IsAllowedToCalculate).Average(y => y.CpuUtilization) 174 }) 175 .ToList(); 176 }); 177 var statusList = new List<DT.Status>(); 178 var e = clientInfos.GetEnumerator(); 179 do { 180 var status = new DT.Status { 181 CoreStatus = new DT.CoreStatus(), 182 CpuUtilizationStatus = new DT.CpuUtilizationStatus(), 183 MemoryStatus = new DT.MemoryStatus() 141 184 }; 142 int freeCores = 0; 143 int freeMemory = 0; 144 int i = 1; 145 foreach (var statistic in statistics) { 146 status.CoreStatus.TotalCores += statistic.SlaveStatistics.Sum(x => x.Cores); 147 freeCores += statistic.SlaveStatistics.Sum(x => x.FreeCores); 148 status.CpuUtilizationStatus.TotalCpuUtilization += statistic.SlaveStatistics.Any() 149 ? statistic.SlaveStatistics.Average(x => x.CpuUtilization) 150 : 0.0; 151 status.MemoryStatus.TotalMemory += statistic.SlaveStatistics.Sum(x => x.Memory); 152 freeMemory += statistic.SlaveStatistics.Sum(x => x.FreeMemory); 153 if (i >= increment) { 154 status.Timestamp = JavascriptUtils.ToTimestamp(statistic.Timestamp); 155 status.CoreStatus.TotalCores /= i; 156 freeCores /= i; 157 status.CpuUtilizationStatus.TotalCpuUtilization /= i; 158 status.MemoryStatus.TotalMemory /= i; 159 freeMemory /= i; 160 status.CoreStatus.ActiveCores = status.CoreStatus.TotalCores; 161 status.MemoryStatus.ActiveMemory = status.MemoryStatus.TotalMemory; 162 status.CpuUtilizationStatus.ActiveCpuUtilization = status.CpuUtilizationStatus.TotalCpuUtilization; 163 status.CpuUtilizationStatus.CalculatingCpuUtilization = status.CpuUtilizationStatus.CalculatingCpuUtilization; 164 status.CoreStatus.CalculatingCores = status.CoreStatus.TotalCores  freeCores; 165 status.MemoryStatus.UsedMemory = status.MemoryStatus.TotalMemory  freeMemory; 166 yield return status; 167 status = new DTO.Status { 168 CoreStatus = new DTO.CoreStatus(), 169 CpuUtilizationStatus = new DTO.CpuUtilizationStatus(), 170 MemoryStatus = new DTO.MemoryStatus() 171 }; 172 freeCores = 0; 173 freeMemory = 0; 174 i = 1; 175 } else { 176 i++; 177 } 185 int i = 0; 186 DateTime lastTimestamp = DateTime.Now; 187 while (e.MoveNext()) { 188 var clientInfo = e.Current; 189 status.CoreStatus.TotalCores += clientInfo.TotalCores; 190 status.CoreStatus.UsedCores += clientInfo.UsedCores; 191 status.MemoryStatus.TotalMemory += clientInfo.TotalMemory; 192 status.MemoryStatus.UsedMemory += clientInfo.UsedMemory; 193 status.CpuUtilizationStatus.TotalCpuUtilization += clientInfo.CpuUtilization; 194 lastTimestamp = clientInfo.Timestamp; 195 i++; 196 if (i >= increment) 197 break; 178 198 } 199 if (i <= 0) continue; 200 status.Timestamp = JavascriptUtils.ToTimestamp(lastTimestamp); 201 status.CoreStatus.TotalCores /= i; 202 status.CoreStatus.UsedCores /= i; 203 status.MemoryStatus.TotalMemory /= i; 204 status.MemoryStatus.UsedMemory /= i; 205 status.CpuUtilizationStatus.TotalCpuUtilization /= i; 206 statusList.Add(status); 207 } while (e.Current != null); 208 return statusList.OrderBy(x => x.Timestamp).ToList(); 209 } 210 211 private double StandardDeviation(IEnumerable<double> source) { 212 int n = 0; 213 double mean = 0; 214 double M2 = 0; 215 foreach (double x in source) { 216 n = n + 1; 217 double delta = x  mean; 218 mean = mean + delta / n; 219 M2 += delta * (x  mean); 179 220 } 221 if (n < 2) { 222 return M2; 223 } 224 return Math.Sqrt(M2 / (n  1)); 180 225 } 181 226 }
Note: See TracChangeset
for help on using the changeset viewer.