Ignore:
Timestamp:
09/21/15 16:28:38 (7 years ago)
Author:
ascheibe
Message:

#2388 merged r12878, r12879, r12883, r12885, r12913, r12914, r12925, r12932, r12961 into stable

Location:
stable
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • stable

  • stable/HeuristicLab.Services.WebApp.Status/3.3/WebApi/DataController.cs

    r12563 r12962  
    2222using System;
    2323using System.Collections.Generic;
    24 using System.Data.Linq;
    2524using System.Linq;
    2625using System.Web.Http;
    2726using HeuristicLab.Services.Hive;
    2827using HeuristicLab.Services.Hive.DataAccess;
    29 using DAL = HeuristicLab.Services.Hive.DataAccess;
    30 using DTO = HeuristicLab.Services.WebApp.Status.WebApi.DataTransfer;
     28using HeuristicLab.Services.Hive.DataAccess.Interfaces;
     29using DT = HeuristicLab.Services.WebApp.Status.WebApi.DataTransfer;
    3130
    3231namespace HeuristicLab.Services.WebApp.Status.WebApi {
    3332  public class DataController : ApiController {
    34 
    35     // start temporary quickfix
     33    private const int LAST_TASKS = 20;
     34
     35    private IPersistenceManager PersistenceManager {
     36      get { return ServiceLocator.Instance.PersistenceManager; }
     37    }
     38
    3639    private const string SQL_USER_TASK_STATUS =
    3740      @"WITH UserTasks AS (
     
    5356    }
    5457
    55     public IEnumerable<DTO.TaskStatus> GetTaskStatus(HiveDataContext db) {
    56       var query = db.ExecuteQuery<UserTaskStatus>(SQL_USER_TASK_STATUS).ToList();
    57       return query.Select(uts => new DTO.TaskStatus {
    58         User = new DTO.User {
    59           Id = uts.UserId.ToString(),
    60           Name = ServiceLocator.Instance.UserManager.GetUserById(uts.UserId).UserName
     58    private IEnumerable<DT.TaskStatus> GetTaskStatus(IPersistenceManager pm) {
     59      return pm.UseTransaction(() => {
     60        var query = pm.DataContext.ExecuteQuery<UserTaskStatus>(SQL_USER_TASK_STATUS).ToList();
     61        return query.Select(uts => new DT.TaskStatus {
     62          User = new DT.User {
     63            Id = uts.UserId.ToString(),
     64            Name = ServiceLocator.Instance.UserManager.GetUserNameById(uts.UserId)
     65          },
     66          CalculatingTasks = uts.CalculatingTasks,
     67          WaitingTasks = uts.WaitingTasks
     68        }).OrderBy(x => x.User.Name);
     69      });
     70    }
     71
     72    private DT.TimeStatus GetTimeStatus(IPersistenceManager pm) {
     73      return pm.UseTransaction(() => {
     74        var factTaskDao = pm.FactTaskDao;
     75        var factTasks = factTaskDao.GetAll();
     76        var completedTasks = factTaskDao.GetCompletedTasks()
     77          .OrderByDescending(x => x.EndTime)
     78          .Take(LAST_TASKS);
     79        var lastCalculatingTimes = completedTasks
     80          .GroupBy(x => 1)
     81          .Select(x => new {
     82            Min = x.Min(y => y.CalculatingTime),
     83            Max = x.Max(y => y.CalculatingTime),
     84            Avg = (long)x.Average(y => (long?)y.CalculatingTime)
     85          }).FirstOrDefault();
     86        var calculatingTasks = factTasks.Where(x => x.TaskState == TaskState.Calculating);
     87        int count = calculatingTasks.Count() / 3;
     88        return new DT.TimeStatus {
     89          MinCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Min : 0,
     90          MaxCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Max : 0,
     91          AvgWaitingTime = count > 0 ? (long)calculatingTasks.OrderBy(x => x.StartTime).Take(count).Average(x => x.InitialWaitingTime) : 0,
     92          AvgCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Avg : 0,
     93          TotalCpuTime = factTasks.Sum(x => Convert.ToInt64(x.CalculatingTime)),
     94          StandardDeviationCalculatingTime = (long)StandardDeviation(completedTasks.Select(x => (double)x.CalculatingTime)),
     95          BeginDate = factTasks.Where(x => x.StartTime.HasValue).OrderBy(x => x.StartTime).Select(x => x.StartTime).FirstOrDefault()
     96        };
     97      });
     98    }
     99
     100    public DT.Status GetStatus() {
     101      var pm = PersistenceManager;
     102      var slaveDao = pm.SlaveDao;
     103      var onlineSlaves = pm.UseTransaction(() => slaveDao.GetOnlineSlaves().ToList());
     104      var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList();
     105      var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList();
     106      int totalCores = onlineSlaves.Sum(s => s.Cores ?? 0);
     107      int totalMemory = onlineSlaves.Sum(s => s.Memory ?? 0);
     108      return new DT.Status {
     109        CoreStatus = new DT.CoreStatus {
     110          TotalCores = totalCores,
     111          UsedCores = totalCores - onlineSlaves.Sum(s => s.FreeCores ?? 0),
     112          ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0),
     113          CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0) - calculatingSlaves.Sum(s => s.FreeCores ?? 0)
    61114        },
    62         CalculatingTasks = uts.CalculatingTasks,
    63         WaitingTasks = uts.WaitingTasks
    64       }).OrderBy(x => x.User.Name);
    65     }
    66     // end temporary quickfix
    67 
    68     public DTO.Status GetStatus() {
    69       using (var db = new HiveDataContext()) {
    70         var onlineSlaves = (from slave in db.Resources.OfType<DAL.Slave>()
    71                             where slave.SlaveState == SlaveState.Calculating || slave.SlaveState == SlaveState.Idle
    72                             select slave).ToList();
    73         var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList();
    74         var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList();
    75         int calculatingMemory = calculatingSlaves.Any() ? (int)calculatingSlaves.Sum(s => s.Memory) : 0;
    76         int freeCalculatingMemory = calculatingSlaves.Any() ? (int)calculatingSlaves.Sum(s => s.FreeMemory) : 0;
    77 
    78         return new DTO.Status {
    79           CoreStatus = new DTO.CoreStatus {
    80             TotalCores = onlineSlaves.Sum(s => s.Cores ?? 0),
    81             FreeCores = onlineSlaves.Sum(s => s.FreeCores ?? 0), // temporary for old chart data
    82             ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0),
    83             CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0) - calculatingSlaves.Sum(s => s.FreeCores ?? 0)
     115        CpuUtilizationStatus = new DT.CpuUtilizationStatus {
     116          TotalCpuUtilization = onlineSlaves.Any()
     117                                ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2)
     118                                : 0.0,
     119          ActiveCpuUtilization = activeSlaves.Any()
     120                                 ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2)
     121                                 : 0.0,
     122          CalculatingCpuUtilization = calculatingSlaves.Any()
     123                                      ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2)
     124                                      : 0.0
     125        },
     126        MemoryStatus = new DT.MemoryStatus {
     127          TotalMemory = totalMemory,
     128          UsedMemory = totalMemory - onlineSlaves.Sum(s => s.FreeMemory ?? 0),
     129          ActiveMemory = activeSlaves.Sum(s => s.Memory ?? 0),
     130          CalculatingMemory = calculatingSlaves.Sum(s => s.Memory ?? 0) - calculatingSlaves.Sum(s => s.FreeMemory ?? 0)
     131        },
     132        TimeStatus = GetTimeStatus(pm),
     133        TasksStatus = GetTaskStatus(pm),
     134        SlavesStatus = onlineSlaves.Select(x => new DT.SlaveStatus {
     135          Slave = new DT.Slave {
     136            Id = x.ResourceId.ToString(),
     137            Name = x.Name
    84138          },
    85           CpuUtilizationStatus = new DTO.CpuUtilizationStatus {
    86             TotalCpuUtilization = onlineSlaves.Any()
    87                                   ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2)
    88                                   : 0.0,
    89             ActiveCpuUtilization = activeSlaves.Any()
    90                                    ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2)
    91                                    : 0.0,
    92             CalculatingCpuUtilization = calculatingSlaves.Any()
    93                                         ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2)
    94                                         : 0.0
    95           },
    96           MemoryStatus = new DTO.MemoryStatus {
    97             TotalMemory = onlineSlaves.Any() ? (int)onlineSlaves.Sum(s => s.Memory) : 0,
    98             FreeMemory = onlineSlaves.Any() ? (int)onlineSlaves.Sum(s => s.FreeMemory) : 0,
    99             ActiveMemory = activeSlaves.Any() ? (int)activeSlaves.Sum(s => s.Memory) : 0,
    100             UsedMemory = calculatingMemory - freeCalculatingMemory
    101           },
    102           TasksStatus = GetTaskStatus(db),
    103           SlavesStatus = onlineSlaves.Select(x => new DTO.SlaveStatus {
    104             Slave = new DTO.Slave {
    105               Id = x.ResourceId.ToString(),
    106               Name = x.Name
    107             },
    108             CpuUtilization = x.CpuUtilization,
    109             Cores = x.Cores ?? 0,
    110             FreeCores = x.FreeCores ?? 0,
    111             Memory = x.Memory ?? 0,
    112             FreeMemory = x.FreeMemory ?? 0,
    113             IsAllowedToCalculate = x.IsAllowedToCalculate,
    114             State = x.SlaveState.ToString()
    115           }).OrderBy(x => x.Slave.Name),
    116           Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now)
    117         };
    118       }
    119     }
    120 
    121     public IEnumerable<DTO.Status> GetStatusHistory(DateTime start, DateTime end) {
     139          CpuUtilization = x.CpuUtilization,
     140          Cores = x.Cores ?? 0,
     141          FreeCores = x.FreeCores ?? 0,
     142          Memory = x.Memory ?? 0,
     143          FreeMemory = x.FreeMemory ?? 0,
     144          IsAllowedToCalculate = x.IsAllowedToCalculate,
     145          State = x.SlaveState.ToString()
     146        }).OrderBy(x => x.Slave.Name),
     147        Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now)
     148      };
     149    }
     150
     151    public IEnumerable<DT.Status> GetStatusHistory(DateTime start, DateTime end) {
    122152      TimeSpan ts = end - start;
    123153      int increment = 1;
     
    127157        increment += 5;
    128158      }
    129       using (var db = new HiveDataContext()) {
    130         DataLoadOptions loadOptions = new DataLoadOptions();
    131         loadOptions.LoadWith<Statistics>(o => o.SlaveStatistics);
    132         db.LoadOptions = loadOptions;
    133         db.DeferredLoadingEnabled = false;
    134         var statistics = db.Statistics.Where(s => s.Timestamp >= start && s.Timestamp <= end)
    135                                       .OrderBy(s => s.Timestamp)
    136                                       .ToList();
    137         var status = new DTO.Status {
    138           CoreStatus = new DTO.CoreStatus(),
    139           CpuUtilizationStatus = new DTO.CpuUtilizationStatus(),
    140           MemoryStatus = new DTO.MemoryStatus()
     159      var pm = PersistenceManager;
     160      var factClientInfoDao = pm.FactClientInfoDao;
     161      var clientInfos = pm.UseTransaction(() => {
     162        return factClientInfoDao.GetAll()
     163          .Where(s => s.Time >= start
     164                      && s.Time <= end
     165                      && s.SlaveState != SlaveState.Offline)
     166          .GroupBy(s => s.Time)
     167          .Select(x => new {
     168            Timestamp = x.Key,
     169            TotalCores = x.Sum(y => y.NumTotalCores),
     170            UsedCores = x.Sum(y => y.NumUsedCores),
     171            TotalMemory = x.Sum(y => y.TotalMemory),
     172            UsedMemory = x.Sum(y => y.UsedMemory),
     173            CpuUtilization = x.Where(y => y.IsAllowedToCalculate).Average(y => y.CpuUtilization)
     174          })
     175          .ToList();
     176      });
     177      var statusList = new List<DT.Status>();
     178      var e = clientInfos.GetEnumerator();
     179      do {
     180        var status = new DT.Status {
     181          CoreStatus = new DT.CoreStatus(),
     182          CpuUtilizationStatus = new DT.CpuUtilizationStatus(),
     183          MemoryStatus = new DT.MemoryStatus()
    141184        };
    142         int freeCores = 0;
    143         int freeMemory = 0;
    144         int i = 1;
    145         foreach (var statistic in statistics) {
    146           status.CoreStatus.TotalCores += statistic.SlaveStatistics.Sum(x => x.Cores);
    147           freeCores += statistic.SlaveStatistics.Sum(x => x.FreeCores);
    148           status.CpuUtilizationStatus.TotalCpuUtilization += statistic.SlaveStatistics.Any()
    149                                                              ? statistic.SlaveStatistics.Average(x => x.CpuUtilization)
    150                                                              : 0.0;
    151           status.MemoryStatus.TotalMemory += statistic.SlaveStatistics.Sum(x => x.Memory);
    152           freeMemory += statistic.SlaveStatistics.Sum(x => x.FreeMemory);
    153           if (i >= increment) {
    154             status.Timestamp = JavascriptUtils.ToTimestamp(statistic.Timestamp);
    155             status.CoreStatus.TotalCores /= i;
    156             freeCores /= i;
    157             status.CpuUtilizationStatus.TotalCpuUtilization /= i;
    158             status.MemoryStatus.TotalMemory /= i;
    159             freeMemory /= i;
    160             status.CoreStatus.ActiveCores = status.CoreStatus.TotalCores;
    161             status.MemoryStatus.ActiveMemory = status.MemoryStatus.TotalMemory;
    162             status.CpuUtilizationStatus.ActiveCpuUtilization = status.CpuUtilizationStatus.TotalCpuUtilization;
    163             status.CpuUtilizationStatus.CalculatingCpuUtilization = status.CpuUtilizationStatus.CalculatingCpuUtilization;
    164             status.CoreStatus.CalculatingCores = status.CoreStatus.TotalCores - freeCores;
    165             status.MemoryStatus.UsedMemory = status.MemoryStatus.TotalMemory - freeMemory;
    166             yield return status;
    167             status = new DTO.Status {
    168               CoreStatus = new DTO.CoreStatus(),
    169               CpuUtilizationStatus = new DTO.CpuUtilizationStatus(),
    170               MemoryStatus = new DTO.MemoryStatus()
    171             };
    172             freeCores = 0;
    173             freeMemory = 0;
    174             i = 1;
    175           } else {
    176             i++;
    177           }
     185        int i = 0;
     186        DateTime lastTimestamp = DateTime.Now;
     187        while (e.MoveNext()) {
     188          var clientInfo = e.Current;
     189          status.CoreStatus.TotalCores += clientInfo.TotalCores;
     190          status.CoreStatus.UsedCores += clientInfo.UsedCores;
     191          status.MemoryStatus.TotalMemory += clientInfo.TotalMemory;
     192          status.MemoryStatus.UsedMemory += clientInfo.UsedMemory;
     193          status.CpuUtilizationStatus.TotalCpuUtilization += clientInfo.CpuUtilization;
     194          lastTimestamp = clientInfo.Timestamp;
     195          i++;
     196          if (i >= increment)
     197            break;
    178198        }
     199        if (i <= 0) continue;
     200        status.Timestamp = JavascriptUtils.ToTimestamp(lastTimestamp);
     201        status.CoreStatus.TotalCores /= i;
     202        status.CoreStatus.UsedCores /= i;
     203        status.MemoryStatus.TotalMemory /= i;
     204        status.MemoryStatus.UsedMemory /= i;
     205        status.CpuUtilizationStatus.TotalCpuUtilization /= i;
     206        statusList.Add(status);
     207      } while (e.Current != null);
     208      return statusList.OrderBy(x => x.Timestamp).ToList();
     209    }
     210
     211    private double StandardDeviation(IEnumerable<double> source) {
     212      int n = 0;
     213      double mean = 0;
     214      double M2 = 0;
     215      foreach (double x in source) {
     216        n = n + 1;
     217        double delta = x - mean;
     218        mean = mean + delta / n;
     219        M2 += delta * (x - mean);
    179220      }
     221      if (n < 2) {
     222        return M2;
     223      }
     224      return Math.Sqrt(M2 / (n - 1));
    180225    }
    181226  }
Note: See TracChangeset for help on using the changeset viewer.