Free cookie consent management tool by TermsFeed Policy Generator

source: branches/crossvalidation-2434/HeuristicLab.Services.WebApp.Status/3.3/WebApi/DataController.cs @ 12935

Last change on this file since 12935 was 12935, checked in by gkronber, 9 years ago

#2434 merged r12932:12934 from trunk to branch

File size: 9.7 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Linq;
25using System.Web.Http;
26using HeuristicLab.Services.Hive;
27using HeuristicLab.Services.Hive.DataAccess;
28using HeuristicLab.Services.Hive.DataAccess.Interfaces;
29using DT = HeuristicLab.Services.WebApp.Status.WebApi.DataTransfer;
30
31namespace HeuristicLab.Services.WebApp.Status.WebApi {
32  public class DataController : ApiController {
33    private const int LAST_TASKS = 20;
34
35    private IPersistenceManager PersistenceManager {
36      get { return ServiceLocator.Instance.PersistenceManager; }
37    }
38
39    private const string SQL_USER_TASK_STATUS =
40      @"WITH UserTasks AS (
41          SELECT Job.OwnerUserId AS UserId, TaskState, COUNT(Task.TaskId) AS Count
42          FROM Task, Job
43          WHERE Task.JobId = Job.JobId AND TaskState IN ('Calculating', 'Waiting')
44          GROUP BY Job.OwnerUserId, TaskState
45        )
46        SELECT
47          DISTINCT UserId,
48          ISNULL((SELECT Count FROM UserTasks WHERE TaskState = 'Calculating' AND UserId = ut.UserId), 0) AS CalculatingTasks,
49          ISNULL((SELECT Count FROM UserTasks WHERE TaskState = 'Waiting' AND UserId = ut.UserId), 0) AS WaitingTasks
50        FROM UserTasks ut;";
51
52    private class UserTaskStatus {
53      public Guid UserId { get; set; }
54      public int CalculatingTasks { get; set; }
55      public int WaitingTasks { get; set; }
56    }
57
58    private IEnumerable<DT.TaskStatus> GetTaskStatus(IPersistenceManager pm) {
59      return pm.UseTransaction(() => {
60        var query = pm.DataContext.ExecuteQuery<UserTaskStatus>(SQL_USER_TASK_STATUS).ToList();
61        return query.Select(uts => new DT.TaskStatus {
62          User = new DT.User {
63            Id = uts.UserId.ToString(),
64            Name = ServiceLocator.Instance.UserManager.GetUserNameById(uts.UserId)
65          },
66          CalculatingTasks = uts.CalculatingTasks,
67          WaitingTasks = uts.WaitingTasks
68        }).OrderBy(x => x.User.Name);
69      });
70    }
71
72    private DT.TimeStatus GetTimeStatus(IPersistenceManager pm) {
73      return pm.UseTransaction(() => {
74        var factTaskDao = pm.FactTaskDao;
75        var factTasks = factTaskDao.GetAll();
76        var completedTasks = factTaskDao.GetCompletedTasks()
77          .OrderByDescending(x => x.EndTime)
78          .Take(LAST_TASKS);
79        var lastCalculatingTimes = completedTasks
80          .GroupBy(x => 1)
81          .Select(x => new {
82            Min = x.Min(y => y.CalculatingTime),
83            Max = x.Max(y => y.CalculatingTime),
84            Avg = (long)x.Average(y => (long?)y.CalculatingTime)
85          }).FirstOrDefault();
86        var calculatingTasks = factTasks.Where(x => x.TaskState == TaskState.Calculating);
87        int count = calculatingTasks.Count() / 3;
88        return new DT.TimeStatus {
89          MinCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Min : 0,
90          MaxCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Max : 0,
91          AvgWaitingTime = count > 0 ? (long)calculatingTasks.OrderBy(x => x.StartTime).Take(count).Average(x => x.InitialWaitingTime) : 0,
92          AvgCalculatingTime = lastCalculatingTimes != null ? lastCalculatingTimes.Avg : 0,
93          TotalCpuTime = factTasks.Sum(x => Convert.ToInt64(x.CalculatingTime)),
94          StandardDeviationCalculatingTime = (long)StandardDeviation(completedTasks.Select(x => (double)x.CalculatingTime)),
95          BeginDate = factTasks.Where(x => x.StartTime.HasValue).OrderBy(x => x.StartTime).Select(x => x.StartTime).FirstOrDefault()
96        };
97      });
98    }
99
100    public DT.Status GetStatus() {
101      var pm = PersistenceManager;
102      var slaveDao = pm.SlaveDao;
103      var onlineSlaves = pm.UseTransaction(() => slaveDao.GetOnlineSlaves().ToList());
104      var activeSlaves = onlineSlaves.Where(s => s.IsAllowedToCalculate).ToList();
105      var calculatingSlaves = activeSlaves.Where(s => s.SlaveState == SlaveState.Calculating).ToList();
106      int totalCores = onlineSlaves.Sum(s => s.Cores ?? 0);
107      int totalMemory = onlineSlaves.Sum(s => s.Memory ?? 0);
108      return new DT.Status {
109        CoreStatus = new DT.CoreStatus {
110          TotalCores = totalCores,
111          UsedCores = totalCores - onlineSlaves.Sum(s => s.FreeCores ?? 0),
112          ActiveCores = activeSlaves.Sum(s => s.Cores ?? 0),
113          CalculatingCores = calculatingSlaves.Sum(s => s.Cores ?? 0) - calculatingSlaves.Sum(s => s.FreeCores ?? 0)
114        },
115        CpuUtilizationStatus = new DT.CpuUtilizationStatus {
116          TotalCpuUtilization = onlineSlaves.Any()
117                                ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2)
118                                : 0.0,
119          ActiveCpuUtilization = activeSlaves.Any()
120                                 ? Math.Round(activeSlaves.Average(s => s.CpuUtilization), 2)
121                                 : 0.0,
122          CalculatingCpuUtilization = calculatingSlaves.Any()
123                                      ? Math.Round(calculatingSlaves.Average(s => s.CpuUtilization), 2)
124                                      : 0.0
125        },
126        MemoryStatus = new DT.MemoryStatus {
127          TotalMemory = totalMemory,
128          UsedMemory = totalMemory - onlineSlaves.Sum(s => s.FreeMemory ?? 0),
129          ActiveMemory = activeSlaves.Sum(s => s.Memory ?? 0),
130          CalculatingMemory = calculatingSlaves.Sum(s => s.Memory ?? 0) - calculatingSlaves.Sum(s => s.FreeMemory ?? 0)
131        },
132        TimeStatus = GetTimeStatus(pm),
133        TasksStatus = GetTaskStatus(pm),
134        SlavesStatus = onlineSlaves.Select(x => new DT.SlaveStatus {
135          Slave = new DT.Slave {
136            Id = x.ResourceId.ToString(),
137            Name = x.Name
138          },
139          CpuUtilization = x.CpuUtilization,
140          Cores = x.Cores ?? 0,
141          FreeCores = x.FreeCores ?? 0,
142          Memory = x.Memory ?? 0,
143          FreeMemory = x.FreeMemory ?? 0,
144          IsAllowedToCalculate = x.IsAllowedToCalculate,
145          State = x.SlaveState.ToString()
146        }).OrderBy(x => x.Slave.Name),
147        Timestamp = JavascriptUtils.ToTimestamp(DateTime.Now)
148      };
149    }
150
151    public IEnumerable<DT.Status> GetStatusHistory(DateTime start, DateTime end) {
152      TimeSpan ts = end - start;
153      int increment = 1;
154      double totalMinutes = ts.TotalMinutes;
155      while (totalMinutes > 5761) {
156        totalMinutes -= 5761;
157        increment += 5;
158      }
159      var pm = PersistenceManager;
160      var factClientInfoDao = pm.FactClientInfoDao;
161      var clientInfos = pm.UseTransaction(() => {
162        return factClientInfoDao.GetAll()
163          .Where(s => s.Time >= start
164                      && s.Time <= end
165                      && s.SlaveState != SlaveState.Offline)
166          .GroupBy(s => s.Time)
167          .Select(x => new {
168            Timestamp = x.Key,
169            TotalCores = x.Sum(y => y.NumTotalCores),
170            UsedCores = x.Sum(y => y.NumUsedCores),
171            TotalMemory = x.Sum(y => y.TotalMemory),
172            UsedMemory = x.Sum(y => y.UsedMemory),
173            CpuUtilization = x.Where(y => y.IsAllowedToCalculate).Average(y => y.CpuUtilization)
174          })
175          .ToList();
176      });
177      var statusList = new List<DT.Status>();
178      var e = clientInfos.GetEnumerator();
179      do {
180        var status = new DT.Status {
181          CoreStatus = new DT.CoreStatus(),
182          CpuUtilizationStatus = new DT.CpuUtilizationStatus(),
183          MemoryStatus = new DT.MemoryStatus()
184        };
185        int i = 0;
186        DateTime lastTimestamp = DateTime.Now;
187        while (e.MoveNext()) {
188          var clientInfo = e.Current;
189          status.CoreStatus.TotalCores += clientInfo.TotalCores;
190          status.CoreStatus.UsedCores += clientInfo.UsedCores;
191          status.MemoryStatus.TotalMemory += clientInfo.TotalMemory;
192          status.MemoryStatus.UsedMemory += clientInfo.UsedMemory;
193          status.CpuUtilizationStatus.TotalCpuUtilization += clientInfo.CpuUtilization;
194          lastTimestamp = clientInfo.Timestamp;
195          i++;
196          if (i >= increment)
197            break;
198        }
199        if (i <= 0) continue;
200        status.Timestamp = JavascriptUtils.ToTimestamp(lastTimestamp);
201        status.CoreStatus.TotalCores /= i;
202        status.CoreStatus.UsedCores /= i;
203        status.MemoryStatus.TotalMemory /= i;
204        status.MemoryStatus.UsedMemory /= i;
205        status.CpuUtilizationStatus.TotalCpuUtilization /= i;
206        statusList.Add(status);
207      } while (e.Current != null);
208      return statusList.OrderBy(x => x.Timestamp).ToList();
209    }
210
211    private double StandardDeviation(IEnumerable<double> source) {
212      int n = 0;
213      double mean = 0;
214      double M2 = 0;
215      foreach (double x in source) {
216        n = n + 1;
217        double delta = x - mean;
218        mean = mean + delta / n;
219        M2 += delta * (x - mean);
220      }
221      if (n < 2) {
222        return M2;
223      }
224      return Math.Sqrt(M2 / (n - 1));
225    }
226  }
227}
Note: See TracBrowser for help on using the repository browser.