source: trunk/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs @ 12878

Last change on this file since 12878 was 12878, checked in by ascheibe, 6 years ago

#2388 merged hive statistics branch into trunk

File size: 16.0 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Data.Linq;
25using System.Linq;
26using HeuristicLab.Services.Access.DataAccess;
27using HeuristicLab.Services.Hive.DataAccess;
28using HeuristicLab.Services.Hive.DataAccess.Manager;
29
30namespace HeuristicLab.Services.Hive {
31  public class HiveStatisticsGenerator : IStatisticsGenerator {
32
33    private const string UnknownUserName = "Unknown";
34    private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
35    private static readonly TaskState[] CompletedStates = { TaskState.Finished, TaskState.Aborted, TaskState.Failed };
36
37    public void GenerateStatistics() {
38      using (var pm = new PersistenceManager()) {
39        pm.UseTransaction(() => {
40          UpdateDimUserTable(pm);
41          UpdateDimJobTable(pm);
42          UpdateDimClientsTable(pm);
43          pm.SubmitChanges();
44        });
45
46        DimTime time = null;
47        pm.UseTransaction(() => {
48          time = UpdateDimTimeTable(pm);
49          pm.SubmitChanges();
50        });
51
52        if (time != null) {
53          pm.UseTransaction(() => {
54            UpdateFactClientInfoTable(time, pm);
55            pm.SubmitChanges();
56          });
57
58          pm.UseTransaction(() => {
59            UpdateTaskFactsTable(time, pm);
60            try {
61              pm.SubmitChanges();
62              UpdateExistingDimJobs(pm);
63              pm.SubmitChanges();
64            }
65            catch (DuplicateKeyException e) {
66              var logger = LogFactory.GetLogger(typeof(HiveStatisticsGenerator).Namespace);
67              logger.Log(string.Format(
68                @"Propable change from summertime to wintertime, resulting in overlapping times.
69                          On wintertime to summertime change, slave timeouts and a fact gap will occur.
70                          Exception Details: {0}", e));
71            }
72          });
73        }
74      }
75    }
76
77    private DimTime UpdateDimTimeTable(PersistenceManager pm) {
78      var dimTimeDao = pm.DimTimeDao;
79      var now = DateTime.Now;
80      var timeEntry = new DimTime {
81        Time = now,
82        Minute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
83        Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
84        Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
85        Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
86        Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
87      };
88      return dimTimeDao.Save(timeEntry);
89    }
90
91    private void UpdateDimUserTable(PersistenceManager pm) {
92      var dimUserDao = pm.DimUserDao;
93      var resourceDao = pm.ResourceDao;
94      var jobDao = pm.JobDao;
95      var existingUserIds = dimUserDao.GetAll().Select(x => x.UserId);
96      var vaildResourceOwnerIds = resourceDao.GetResourcesWithValidOwner().Select(x => x.OwnerUserId.Value);
97      var jobOwnerIds = jobDao.GetAll().Select(x => x.OwnerUserId);
98      var newUserIds = vaildResourceOwnerIds
99        .Union(jobOwnerIds)
100        .Where(id => !existingUserIds.Contains(id))
101        .ToList();
102      dimUserDao.Save(newUserIds.Select(x => new DimUser {
103        UserId = x,
104        Name = GetUserName(x)
105      }));
106    }
107
108    private void UpdateDimJobTable(PersistenceManager pm) {
109      var dimJobDao = pm.DimJobDao;
110      var jobDao = pm.JobDao;
111      var taskDao = pm.TaskDao;
112      var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId);
113      var newJobs = jobDao.GetAll()
114        .Where(x => !dimJobIds.Contains(x.JobId))
115        .Select(x => new {
116          JobId = x.JobId,
117          UserId = x.OwnerUserId,
118          JobName = x.Name ?? string.Empty,
119          DateCreated = x.DateCreated,
120          TotalTasks = taskDao.GetAll().Count(y => y.JobId == x.JobId)
121        })
122        .ToList();
123      dimJobDao.Save(newJobs.Select(x => new DimJob {
124        JobId = x.JobId,
125        JobName = x.JobName,
126        UserId = x.UserId,
127        UserName = GetUserName(x.UserId),
128        DateCreated = x.DateCreated,
129        TotalTasks = x.TotalTasks,
130        CompletedTasks = 0,
131        DateCompleted = null
132      }));
133    }
134
135    private void UpdateExistingDimJobs(PersistenceManager pm) {
136      var jobDao = pm.JobDao;
137      var dimJobDao = pm.DimJobDao;
138      var factTaskDao = pm.FactTaskDao;
139      foreach (var dimJob in dimJobDao.GetNotCompletedJobs()) {
140        var taskStates = factTaskDao.GetByJobId(dimJob.JobId)
141            .GroupBy(x => x.TaskState)
142            .Select(x => new {
143              State = x.Key,
144              Count = x.Count()
145            }).ToList();
146        int totalTasks = 0, completedTasks = 0;
147        foreach (var state in taskStates) {
148          totalTasks += state.Count;
149          if (CompletedStates.Contains(state.State)) {
150            completedTasks += state.Count;
151          }
152        }
153        if (totalTasks == completedTasks) {
154          var completeDate = factTaskDao.GetLastCompletedTaskFromJob(dimJob.JobId);
155          if (completeDate == null) {
156            if (jobDao.GetById(dimJob.JobId) == null) {
157              completeDate = DateTime.Now;
158            }
159          }
160          dimJob.DateCompleted = completeDate;
161        }
162        dimJob.TotalTasks = totalTasks;
163        dimJob.CompletedTasks = completedTasks;
164      }
165    }
166
167    private void UpdateDimClientsTable(PersistenceManager pm) {
168      var dimClientDao = pm.DimClientDao;
169      var slaveDao = pm.SlaveDao;
170      var slaves = slaveDao.GetAll();
171      var recentlyAddedClients = dimClientDao.GetActiveClients();
172      var slaveIds = slaves.Select(x => x.ResourceId);
173
174      var removedClientIds = recentlyAddedClients
175        .Where(x => !slaveIds.Contains(x.ResourceId))
176        .Select(x => x.Id);
177      var modifiedClients =
178        from slave in slaves
179        join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
180        where (slave.Name != client.Name
181               || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
182               || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
183               || slave.ParentResourceId != client.ResourceGroupId
184               || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
185        select new {
186          SlaveId = slave.ResourceId,
187          ClientId = client.Id
188        };
189      var clientIds = dimClientDao.GetActiveClients().Select(x => x.ResourceId);
190      var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
191      var newClients = slaves
192        .Where(x => !clientIds.Contains(x.ResourceId)
193                    || modifiedClientIds.Contains(x.ResourceId))
194        .Select(x => new {
195          x.ResourceId,
196          x.Name,
197          ResourceGroupId = x.ParentResourceId,
198          GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
199          ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
200          GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
201        })
202        .ToList();
203
204      var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
205      dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
206      dimClientDao.Save(newClients.Select(x => new DimClient {
207        ResourceId = x.ResourceId,
208        Name = x.Name,
209        ExpirationTime = null,
210        ResourceGroupId = x.ResourceGroupId,
211        GroupName = x.GroupName,
212        ResourceGroup2Id = x.ResourceGroup2Id,
213        GroupName2 = x.GroupName2
214      }));
215    }
216
217    private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
218      var factClientInfoDao = pm.FactClientInfoDao;
219      var slaveDao = pm.SlaveDao;
220      var dimClientDao = pm.DimClientDao;
221
222      var newRawFactInfos =
223        from s in slaveDao.GetAll()
224        join c in dimClientDao.GetActiveClients() on s.ResourceId equals c.ResourceId
225        join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
226        from cf in joinCf.DefaultIfEmpty()
227        select new {
228          ClientId = c.Id,
229          UserId = s.OwnerUserId ?? Guid.Empty,
230          TotalCores = s.Cores ?? 0,
231          FreeCores = s.FreeCores ?? 0,
232          TotalMemory = s.Memory ?? 0,
233          FreeMemory = s.FreeMemory ?? 0,
234          CpuUtilization = s.CpuUtilization,
235          SlaveState = s.SlaveState,
236          IsAllowedToCalculate = s.IsAllowedToCalculate,
237          LastFactTimestamp = cf.Timestamp
238        };
239
240      factClientInfoDao.Save(
241        from x in newRawFactInfos.ToList()
242        let duration = x.LastFactTimestamp != null
243                       ? (int)(newTime.Time - (DateTime)x.LastFactTimestamp).TotalSeconds
244                       : (int)SmallestTimeSpan.TotalSeconds
245        select new FactClientInfo {
246          ClientId = x.ClientId,
247          DimTime = newTime,
248          UserId = x.UserId,
249          NumUsedCores = x.TotalCores - x.FreeCores,
250          NumTotalCores = x.TotalCores,
251          UsedMemory = x.TotalMemory - x.FreeMemory,
252          TotalMemory = x.TotalMemory,
253          CpuUtilization = Math.Round(x.CpuUtilization, 2),
254          SlaveState = x.SlaveState,
255          IdleTime = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0,
256          UnavailableTime = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0,
257          OfflineTime = x.SlaveState == SlaveState.Offline ? duration : 0,
258          IsAllowedToCalculate = x.IsAllowedToCalculate
259        }
260      );
261    }
262
263    private void UpdateTaskFactsTable(DimTime newTime, PersistenceManager pm) {
264      var factTaskDao = pm.FactTaskDao;
265      var taskDao = pm.TaskDao;
266      var dimClientDao = pm.DimClientDao;
267
268      var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
269      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
270        x.TaskId,
271        x.LastClientId
272      });
273
274      var newTasks =
275        from task in taskDao.GetAllChildTasks()
276        let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
277        let lastSlaveId = stateLogs.First(x => x.SlaveId != null).SlaveId
278        where (!factTaskIds.Contains(task.TaskId)
279               || notFinishedFactTasks.Select(x => x.TaskId).Contains(task.TaskId))
280        join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
281        from lastFact in lastFactPerTask.DefaultIfEmpty()
282        join client in dimClientDao.GetActiveClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
283        from client in clientsPerSlaveId.DefaultIfEmpty()
284        select new {
285          TaskId = task.TaskId,
286          JobId = task.JobId,
287          Priority = task.Priority,
288          CoresRequired = task.CoresNeeded,
289          MemoryRequired = task.MemoryNeeded,
290          State = task.State,
291          StateLogs = stateLogs.OrderBy(x => x.DateTime),
292          LastClientId = client != null
293                         ? client.Id : lastFact != null
294                         ? lastFact.LastClientId : (Guid?)null
295        };
296      factTaskDao.Save(
297        from x in newTasks.ToList()
298        let taskData = CalculateFactTaskData(x.StateLogs)
299        select new FactTask {
300          TaskId = x.TaskId,
301          JobId = x.JobId,
302          StartTime = taskData.StartTime,
303          EndTime = taskData.EndTime,
304          LastClientId = x.LastClientId,
305          Priority = x.Priority,
306          CoresRequired = x.CoresRequired,
307          MemoryRequired = x.MemoryRequired,
308          NumCalculationRuns = taskData.CalculationRuns,
309          NumRetries = taskData.Retries,
310          WaitingTime = taskData.WaitingTime,
311          CalculatingTime = taskData.CalculatingTime,
312          TransferTime = taskData.TransferTime,
313          TaskState = x.State,
314          Exception = taskData.Exception,
315          InitialWaitingTime = taskData.InitialWaitingTime
316        });
317      factTaskDao.Delete(notFinishedFactTasks.Select(x => x.TaskId));
318    }
319
320    private string GetUserName(Guid userId) {
321      try {
322        // we cannot use the ServiceLocator.Instance.UserManager since the janitor service
323        // is not hosted in the iis the MemberShip.GetUser method causes exceptions
324        // needs to be further investigated current workaround: use the authenticationcontext
325        // we could also connect to the access service to get the user name
326        using (ASPNETAuthenticationDataContext dc = new ASPNETAuthenticationDataContext()) {
327          var user = dc.aspnet_Users.SingleOrDefault(x => x.UserId == userId);
328          return user != null ? user.UserName : UnknownUserName;
329        }
330      }
331      catch (Exception) {
332        return UnknownUserName;
333      }
334    }
335
336    private class FactTaskData {
337      public int CalculationRuns { get; set; }
338      public int Retries { get; set; }
339      public long CalculatingTime { get; set; }
340      public long WaitingTime { get; set; }
341      public long TransferTime { get; set; }
342      public long InitialWaitingTime { get; set; }
343      public string Exception { get; set; }
344      public DateTime? StartTime { get; set; }
345      public DateTime? EndTime { get; set; }
346    }
347
348    private FactTaskData CalculateFactTaskData(IEnumerable<StateLog> stateLogs) {
349      var factTaskData = new FactTaskData();
350      var enumerator = stateLogs.GetEnumerator();
351      if (enumerator.MoveNext()) {
352        StateLog current = enumerator.Current, first = current, prev = null;
353        while (current != null) {
354          var next = enumerator.MoveNext() ? enumerator.Current : null;
355          int timeSpanInSeconds;
356          if (next != null) {
357            timeSpanInSeconds = (int)(next.DateTime - current.DateTime).TotalSeconds;
358          } else {
359            timeSpanInSeconds = (int)(DateTime.Now - current.DateTime).TotalSeconds;
360            factTaskData.Exception = current.Exception;
361          }
362          switch (current.State) {
363            case TaskState.Calculating:
364              factTaskData.CalculatingTime += timeSpanInSeconds;
365              factTaskData.CalculationRuns++;
366              if (factTaskData.CalculationRuns == 1) {
367                factTaskData.StartTime = current.DateTime;
368                factTaskData.InitialWaitingTime = (int)(current.DateTime - first.DateTime).TotalSeconds;
369              }
370              if (prev != null && prev.State != TaskState.Transferring) {
371                factTaskData.Retries++;
372              }
373              break;
374
375            case TaskState.Waiting:
376              factTaskData.WaitingTime += timeSpanInSeconds;
377              break;
378
379            case TaskState.Transferring:
380              factTaskData.TransferTime += timeSpanInSeconds;
381              break;
382
383            case TaskState.Finished:
384            case TaskState.Failed:
385            case TaskState.Aborted:
386              factTaskData.EndTime = current.DateTime;
387              break;
388          }
389          prev = current;
390          current = next;
391        }
392      }
393      return factTaskData;
394    }
395  }
396}
Note: See TracBrowser for help on using the repository browser.