1  #region License Information


2  /* HeuristicLab


3  * Copyright (C) 20022016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)


4  *


5  * This file is part of HeuristicLab.


6  *


7  * HeuristicLab is free software: you can redistribute it and/or modify


8  * it under the terms of the GNU General Public License as published by


9  * the Free Software Foundation, either version 3 of the License, or


10  * (at your option) any later version.


11  *


12  * HeuristicLab is distributed in the hope that it will be useful,


13  * but WITHOUT ANY WARRANTY; without even the implied warranty of


14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the


15  * GNU General Public License for more details.


16  *


17  * You should have received a copy of the GNU General Public License


18  * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.


19  */


20  #endregion


21 


22  using System;


23  using System.Collections.Generic;


24  using System.Linq;


25  using System.Web.Http;


26  using HeuristicLab.Services.Hive;


27  using HeuristicLab.Services.Hive.DataAccess;


28  using HeuristicLab.Services.Hive.DataAccess.Interfaces;


29  using DT = HeuristicLab.Services.WebApp.Status.WebApi.DataTransfer;


30 


31  namespace HeuristicLab.Services.WebApp.Status.WebApi {


32  public class DataController : ApiController {


33  private const int LAST_TASKS = 20;


34 


35  private IPersistenceManager PersistenceManager {


36  get { return ServiceLocator.Instance.PersistenceManager; }


37  }


38 


39  private const string SQL_USER_TASK_STATUS =


40  @"WITH UserTasks AS (


41  SELECT Job.OwnerUserId AS UserId, TaskState, COUNT(Task.TaskId) AS Count


42  FROM Task, Job


43  WHERE Task.JobId = Job.JobId AND TaskState IN ('Calculating', 'Waiting')


44  GROUP BY Job.OwnerUserId, TaskState


45  )


46  SELECT


47  DISTINCT UserId,


48  ISNULL((SELECT Count FROM UserTasks WHERE TaskState = 'Calculating' AND UserId = ut.UserId), 0) AS CalculatingTasks,


49  ISNULL((SELECT Count FROM UserTasks WHERE TaskState = 'Waiting' AND UserId = ut.UserId), 0) AS WaitingTasks


50  FROM UserTasks ut;";


51 


52  private class UserTaskStatus {


53  public Guid UserId { get; set; }


54  public int CalculatingTasks { get; set; }


55  public int WaitingTasks { get; set; }


56  }


57 


58  private IEnumerable<DT.TaskStatus> GetTaskStatus(IPersistenceManager pm) {


59  return pm.UseTransaction(() => {


60  var query = pm.DataContext.ExecuteQuery<UserTaskStatus>(SQL_USER_TASK_STATUS).ToList();


61  return query.Select(uts => new DT.TaskStatus {


62  User = new DT.User {


63  Id = uts.UserId.ToString(),


64  Name = ServiceLocator.Instance.UserManager.GetUserNameById(uts.UserId)


65  },


66  CalculatingTasks = uts.CalculatingTasks,


67  WaitingTasks = uts.WaitingTasks


68  }).OrderBy(x => x.User.Name);


69  });


70  }


71 


72  private DT.TimeStatus GetTimeStatus(IPersistenceManager pm) {


73  return pm.UseTransaction(() => {


74  var factTaskDao = pm.FactTaskDao;


75  var factTasks = factTaskDao.GetAll();


76  var completedTasks = factTaskDao.GetCompletedTasks()


77  .OrderByDescending(x => x.EndTime)


78  .Take(LAST_TASKS);


79  var lastCalculatingTimes = completedTasks


80  .GroupBy(x => 1)


81  .Select(x => new {


82  Min = x.Min(y => y.CalculatingTime),


83  Max = x.Max(y => y.CalculatingTime),


84  Avg = (long)x.Average(y => (long?)y.CalculatingTime)


85  }).FirstOrDefault();


86  var calculatingTasks = factTasks.Where(x => x.TaskState == TaskState.Calculating);


87  int count = calculatingTasks.Count() / 3;


88  return new DT.TimeStatus {


89  MinCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Min : 0,


90  MaxCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Max : 0,


91  AvgWaitingTime = count > 0 ? (long)calculatingTasks.OrderBy(x => x.StartTime).Take(count).Average(x => x.InitialWaitingTime) : 0,


92  AvgCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Avg : 0,


93  TotalCpuTime = factTasks.Sum(x => (long?)x.CalculatingTime) ?? 0,


94  StandardDeviationCalculatingTime = (long)StandardDeviation(completedTasks.Select(x => (double)x.CalculatingTime)),


95  BeginDate = factTasks.Where(x => x.StartTime.HasValue).OrderBy(x => x.StartTime).Select(x => x.StartTime).FirstOrDefault()


96  };


97  });


98  }


99 


100  public DT.Status GetStatus() {


101  var pm = PersistenceManager;


102  var slaveDao = pm.SlaveDao;


103  var onlineSlaves = pm.UseTransaction(() => slaveDao.GetOnlineSlaves().ToList());


104  var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList();


105  var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList();


106  int totalCores = onlineSlaves.Sum(s => s.Cores ?? 0);


107  int totalMemory = onlineSlaves.Sum(s => s.Memory ?? 0);


108  return new DT.Status {


109  CoreStatus = new DT.CoreStatus {


110  TotalCores = totalCores,


111  UsedCores = totalCores  onlineSlaves.Sum(s => s.FreeCores ?? 0),


112  ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0),


113  CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0)  calculatingSlaves.Sum(s => s.FreeCores ?? 0)


114  },


115  CpuUtilizationStatus = new DT.CpuUtilizationStatus {


116  TotalCpuUtilization = onlineSlaves.Any()


117  ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2)


118  : 0.0,


119  ActiveCpuUtilization = activeSlaves.Any()


120  ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2)


121  : 0.0,


122  CalculatingCpuUtilization = calculatingSlaves.Any()


123  ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2)


124  : 0.0


125  },


126  MemoryStatus = new DT.MemoryStatus {


127  TotalMemory = totalMemory,


128  UsedMemory = totalMemory  onlineSlaves.Sum(s => s.FreeMemory ?? 0),


129  ActiveMemory = activeSlaves.Sum(s => s.Memory ?? 0),


130  CalculatingMemory = calculatingSlaves.Sum(s => s.Memory ?? 0)  calculatingSlaves.Sum(s => s.FreeMemory ?? 0)


131  },


132  TimeStatus = GetTimeStatus(pm),


133  TasksStatus = GetTaskStatus(pm),


134  SlavesStatus = onlineSlaves.Select(x => new DT.SlaveStatus {


135  Slave = new DT.Slave {


136  Id = x.ResourceId.ToString(),


137  Name = x.Name


138  },


139  CpuUtilization = x.CpuUtilization,


140  Cores = x.Cores ?? 0,


141  FreeCores = x.FreeCores ?? 0,


142  Memory = x.Memory ?? 0,


143  FreeMemory = x.FreeMemory ?? 0,


144  IsAllowedToCalculate = x.IsAllowedToCalculate,


145  State = x.SlaveState.ToString()


146  }).OrderBy(x => x.Slave.Name),


147  Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now)


148  };


149  }


150 


151  public IEnumerable<DT.Status> GetStatusHistory(DateTime start, DateTime end) {


152  TimeSpan ts = end  start;


153  int increment = 1;


154  double totalMinutes = ts.TotalMinutes;


155  while (totalMinutes > 5761) {


156  totalMinutes = 5761;


157  increment += 5;


158  }


159  var pm = PersistenceManager;


160  var factClientInfoDao = pm.FactClientInfoDao;


161  var clientInfos = pm.UseTransaction(() => {


162  return factClientInfoDao.GetAll()


163  .Where(s => s.Time >= start


164  && s.Time <= end


165  && s.SlaveState != SlaveState.Offline)


166  .GroupBy(s => s.Time)


167  .Select(x => new {


168  Timestamp = x.Key,


169  TotalCores = x.Sum(y => y.NumTotalCores),


170  UsedCores = x.Sum(y => y.NumUsedCores),


171  TotalMemory = x.Sum(y => y.TotalMemory),


172  UsedMemory = x.Sum(y => y.UsedMemory),


173  CpuUtilization = x.Where(y => y.IsAllowedToCalculate).Average(y => y.CpuUtilization)


174  })


175  .ToList();


176  });


177  var statusList = new List<DT.Status>();


178  var e = clientInfos.GetEnumerator();


179  do {


180  var status = new DT.Status {


181  CoreStatus = new DT.CoreStatus(),


182  CpuUtilizationStatus = new DT.CpuUtilizationStatus(),


183  MemoryStatus = new DT.MemoryStatus()


184  };


185  int i = 0;


186  DateTime lastTimestamp = DateTime.Now;


187  while (e.MoveNext()) {


188  var clientInfo = e.Current;


189  status.CoreStatus.TotalCores += clientInfo.TotalCores;


190  status.CoreStatus.UsedCores += clientInfo.UsedCores;


191  status.MemoryStatus.TotalMemory += clientInfo.TotalMemory;


192  status.MemoryStatus.UsedMemory += clientInfo.UsedMemory;


193  status.CpuUtilizationStatus.TotalCpuUtilization += clientInfo.CpuUtilization;


194  lastTimestamp = clientInfo.Timestamp;


195  i++;


196  if (i >= increment)


197  break;


198  }


199  if (i <= 0) continue;


200  status.Timestamp = JavascriptUtils.ToTimestamp(lastTimestamp);


201  status.CoreStatus.TotalCores /= i;


202  status.CoreStatus.UsedCores /= i;


203  status.MemoryStatus.TotalMemory /= i;


204  status.MemoryStatus.UsedMemory /= i;


205  status.CpuUtilizationStatus.TotalCpuUtilization /= i;


206  statusList.Add(status);


207  } while (e.Current != null);


208  return statusList.OrderBy(x => x.Timestamp).ToList();


209  }


210 


211  private double StandardDeviation(IEnumerable<double> source) {


212  int n = 0;


213  double mean = 0;


214  double M2 = 0;


215  foreach (double x in source) {


216  n = n + 1;


217  double delta = x  mean;


218  mean = mean + delta / n;


219  M2 += delta * (x  mean);


220  }


221  if (n < 2) {


222  return M2;


223  }


224  return Math.Sqrt(M2 / (n  1));


225  }


226  }


227  } 
