source: stable/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs @ 12962

Last change on this file since 12962 was 12962, checked in by ascheibe, 7 years ago

#2388 merged r12878, r12879, r12883, r12885, r12913, r12914, r12925, r12932, r12961 into stable

File size: 17.2 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Data.Linq;
25using System.Linq;
26using HeuristicLab.Services.Access.DataAccess;
27using HeuristicLab.Services.Hive.DataAccess;
28using HeuristicLab.Services.Hive.DataAccess.Manager;
29
30namespace HeuristicLab.Services.Hive {
31  public class HiveStatisticsGenerator : IStatisticsGenerator {
32
33    private const string UnknownUserName = "Unknown";
34    private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
35    private static readonly TaskState[] CompletedStates = { TaskState.Finished, TaskState.Aborted, TaskState.Failed };
36
37    public void GenerateStatistics() {
38      using (var pm = new PersistenceManager()) {
39        pm.UseTransaction(() => {
40          UpdateDimUserTable(pm);
41          UpdateDimJobTable(pm);
42          UpdateDimClientsTable(pm);
43          pm.SubmitChanges();
44        });
45
46        DimTime time = null;
47        pm.UseTransaction(() => {
48          time = UpdateDimTimeTable(pm);
49          pm.SubmitChanges();
50        });
51
52        if (time != null) {
53          pm.UseTransaction(() => {
54            UpdateFactClientInfoTable(time, pm);
55            pm.SubmitChanges();
56          });
57
58          pm.UseTransaction(() => {
59            UpdateTaskFactsTable(pm);
60            try {
61              pm.SubmitChanges();
62              UpdateExistingDimJobs(pm);
63              pm.SubmitChanges();
64            }
65            catch (DuplicateKeyException e) {
66              var logger = LogFactory.GetLogger(typeof(HiveStatisticsGenerator).Namespace);
67              logger.Log(string.Format(
68                @"Propable change from summertime to wintertime, resulting in overlapping times.
69                          On wintertime to summertime change, slave timeouts and a fact gap will occur.
70                          Exception Details: {0}", e));
71            }
72          });
73        }
74      }
75    }
76
77    private DimTime UpdateDimTimeTable(PersistenceManager pm) {
78      var dimTimeDao = pm.DimTimeDao;
79      var now = DateTime.Now;
80      var timeEntry = new DimTime {
81        Time = now,
82        Minute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
83        Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
84        Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
85        Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
86        Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
87      };
88      return dimTimeDao.Save(timeEntry);
89    }
90
91    private void UpdateDimUserTable(PersistenceManager pm) {
92      var dimUserDao = pm.DimUserDao;
93      var resourceDao = pm.ResourceDao;
94      var jobDao = pm.JobDao;
95      var existingUserIds = dimUserDao.GetAll().Select(x => x.UserId);
96      var vaildResourceOwnerIds = resourceDao.GetResourcesWithValidOwner().Select(x => x.OwnerUserId.Value);
97      var jobOwnerIds = jobDao.GetAll().Select(x => x.OwnerUserId);
98      var newUserIds = vaildResourceOwnerIds
99        .Union(jobOwnerIds)
100        .Where(id => !existingUserIds.Contains(id))
101        .ToList();
102      dimUserDao.Save(newUserIds.Select(x => new DimUser {
103        UserId = x,
104        Name = GetUserName(x)
105      }));
106    }
107
108    private void UpdateDimJobTable(PersistenceManager pm) {
109      var dimJobDao = pm.DimJobDao;
110      var jobDao = pm.JobDao;
111      var taskDao = pm.TaskDao;
112      var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId);
113      var newJobs = jobDao.GetAll()
114        .Where(x => !dimJobIds.Contains(x.JobId))
115        .Select(x => new {
116          JobId = x.JobId,
117          UserId = x.OwnerUserId,
118          JobName = x.Name ?? string.Empty,
119          DateCreated = x.DateCreated,
120          TotalTasks = taskDao.GetAll().Count(y => y.JobId == x.JobId)
121        })
122        .ToList();
123      dimJobDao.Save(newJobs.Select(x => new DimJob {
124        JobId = x.JobId,
125        JobName = x.JobName,
126        UserId = x.UserId,
127        UserName = GetUserName(x.UserId),
128        DateCreated = x.DateCreated,
129        TotalTasks = x.TotalTasks,
130        CompletedTasks = 0,
131        DateCompleted = null
132      }));
133    }
134
135    private void UpdateExistingDimJobs(PersistenceManager pm) {
136      var jobDao = pm.JobDao;
137      var dimJobDao = pm.DimJobDao;
138      var factTaskDao = pm.FactTaskDao;
139      foreach (var dimJob in dimJobDao.GetNotCompletedJobs()) {
140        var taskStates = factTaskDao.GetByJobId(dimJob.JobId)
141            .GroupBy(x => x.TaskState)
142            .Select(x => new {
143              State = x.Key,
144              Count = x.Count()
145            }).ToList();
146        int totalTasks = 0, completedTasks = 0;
147        foreach (var state in taskStates) {
148          totalTasks += state.Count;
149          if (CompletedStates.Contains(state.State)) {
150            completedTasks += state.Count;
151          }
152        }
153        if (totalTasks == completedTasks) {
154          var completeDate = factTaskDao.GetLastCompletedTaskFromJob(dimJob.JobId);
155          if (completeDate == null) {
156            if (jobDao.GetById(dimJob.JobId) == null) {
157              completeDate = DateTime.Now;
158            }
159          }
160          dimJob.DateCompleted = completeDate;
161        }
162        dimJob.TotalTasks = totalTasks;
163        dimJob.CompletedTasks = completedTasks;
164      }
165    }
166
167    private void UpdateDimClientsTable(PersistenceManager pm) {
168      var dimClientDao = pm.DimClientDao;
169      var slaveDao = pm.SlaveDao;
170      var slaves = slaveDao.GetAll();
171      var recentlyAddedClients = dimClientDao.GetActiveClients();
172      var slaveIds = slaves.Select(x => x.ResourceId);
173
174      var removedClientIds = recentlyAddedClients
175        .Where(x => !slaveIds.Contains(x.ResourceId))
176        .Select(x => x.Id);
177      var modifiedClients =
178        from slave in slaves
179        join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
180        where (slave.Name != client.Name
181               || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
182               || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
183               || slave.ParentResourceId != client.ResourceGroupId
184               || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
185        select new {
186          SlaveId = slave.ResourceId,
187          ClientId = client.Id
188        };
189      var clientIds = dimClientDao.GetActiveClients().Select(x => x.ResourceId);
190      var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
191      var newClients = slaves
192        .Where(x => !clientIds.Contains(x.ResourceId)
193                    || modifiedClientIds.Contains(x.ResourceId))
194        .Select(x => new {
195          x.ResourceId,
196          x.Name,
197          ResourceGroupId = x.ParentResourceId,
198          GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
199          ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
200          GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
201        })
202        .ToList();
203
204      var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
205      dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
206      dimClientDao.Save(newClients.Select(x => new DimClient {
207        ResourceId = x.ResourceId,
208        Name = x.Name,
209        ExpirationTime = null,
210        ResourceGroupId = x.ResourceGroupId,
211        GroupName = x.GroupName,
212        ResourceGroup2Id = x.ResourceGroup2Id,
213        GroupName2 = x.GroupName2
214      }));
215    }
216
217    private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
218      var factClientInfoDao = pm.FactClientInfoDao;
219      var slaveDao = pm.SlaveDao;
220      var dimClientDao = pm.DimClientDao;
221
222      var newRawFactInfos =
223        from s in slaveDao.GetAll()
224        join c in dimClientDao.GetActiveClients() on s.ResourceId equals c.ResourceId
225        join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
226        from cf in joinCf.DefaultIfEmpty()
227        select new {
228          ClientId = c.Id,
229          UserId = s.OwnerUserId ?? Guid.Empty,
230          TotalCores = s.Cores ?? 0,
231          FreeCores = s.FreeCores ?? 0,
232          TotalMemory = s.Memory ?? 0,
233          FreeMemory = s.FreeMemory ?? 0,
234          CpuUtilization = s.CpuUtilization,
235          SlaveState = s.SlaveState,
236          IsAllowedToCalculate = s.IsAllowedToCalculate,
237          LastFactTimestamp = cf.Timestamp
238        };
239
240      factClientInfoDao.Save(
241        from x in newRawFactInfos.ToList()
242        let duration = x.LastFactTimestamp != null
243                       ? (int)(newTime.Time - (DateTime)x.LastFactTimestamp).TotalSeconds
244                       : (int)SmallestTimeSpan.TotalSeconds
245        select new FactClientInfo {
246          ClientId = x.ClientId,
247          DimTime = newTime,
248          UserId = x.UserId,
249          NumUsedCores = x.TotalCores - x.FreeCores,
250          NumTotalCores = x.TotalCores,
251          UsedMemory = x.TotalMemory - x.FreeMemory,
252          TotalMemory = x.TotalMemory,
253          CpuUtilization = Math.Round(x.CpuUtilization, 2),
254          SlaveState = x.SlaveState,
255          IdleTime = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0,
256          UnavailableTime = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0,
257          OfflineTime = x.SlaveState == SlaveState.Offline ? duration : 0,
258          IsAllowedToCalculate = x.IsAllowedToCalculate
259        }
260      );
261    }
262
263    private void UpdateTaskFactsTable(PersistenceManager pm) {
264      var factTaskDao = pm.FactTaskDao;
265      var taskDao = pm.TaskDao;
266      var dimClientDao = pm.DimClientDao;
267
268      var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
269      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
270        x.TaskId,
271        x.LastClientId
272      });
273
274      var newTasks =
275        (from task in taskDao.GetAllChildTasks()
276         let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
277         let lastSlaveId = stateLogs.First(x => x.SlaveId != null).SlaveId
278         where (!factTaskIds.Contains(task.TaskId)
279                || notFinishedFactTasks.Select(x => x.TaskId).Contains(task.TaskId))
280         join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
281         from lastFact in lastFactPerTask.DefaultIfEmpty()
282         join client in dimClientDao.GetActiveClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
283         from client in clientsPerSlaveId.DefaultIfEmpty()
284         select new {
285           TaskId = task.TaskId,
286           JobId = task.JobId,
287           Priority = task.Priority,
288           CoresRequired = task.CoresNeeded,
289           MemoryRequired = task.MemoryNeeded,
290           State = task.State,
291           StateLogs = stateLogs.OrderBy(x => x.DateTime),
292           LastClientId = client != null
293                          ? client.Id : lastFact != null
294                          ? lastFact.LastClientId : (Guid?)null,
295           NotFinishedTask = notFinishedFactTasks.Any(y => y.TaskId == task.TaskId)
296         }).ToList();
297
298      //insert facts for new tasks
299      factTaskDao.Save(
300        from x in newTasks
301        where !x.NotFinishedTask
302        let taskData = CalculateFactTaskData(x.StateLogs)
303        select new FactTask {
304          TaskId = x.TaskId,
305          JobId = x.JobId,
306          StartTime = taskData.StartTime,
307          EndTime = taskData.EndTime,
308          LastClientId = x.LastClientId,
309          Priority = x.Priority,
310          CoresRequired = x.CoresRequired,
311          MemoryRequired = x.MemoryRequired,
312          NumCalculationRuns = taskData.CalculationRuns,
313          NumRetries = taskData.Retries,
314          WaitingTime = taskData.WaitingTime,
315          CalculatingTime = taskData.CalculatingTime,
316          TransferTime = taskData.TransferTime,
317          TaskState = x.State,
318          Exception = taskData.Exception,
319          InitialWaitingTime = taskData.InitialWaitingTime
320        });
321
322      //update data of already existing facts
323      foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
324        var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
325        if (ntc.Any()) {
326          var x = ntc.Single();
327          var taskData = CalculateFactTaskData(x.StateLogs);
328
329          notFinishedTask.StartTime = taskData.StartTime;
330          notFinishedTask.EndTime = taskData.EndTime;
331          notFinishedTask.LastClientId = x.LastClientId;
332          notFinishedTask.Priority = x.Priority;
333          notFinishedTask.CoresRequired = x.CoresRequired;
334          notFinishedTask.MemoryRequired = x.MemoryRequired;
335          notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
336          notFinishedTask.NumRetries = taskData.Retries;
337          notFinishedTask.WaitingTime = taskData.WaitingTime;
338          notFinishedTask.CalculatingTime = taskData.CalculatingTime;
339          notFinishedTask.TransferTime = taskData.TransferTime;
340          notFinishedTask.TaskState = x.State;
341          notFinishedTask.Exception = taskData.Exception;
342          notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
343        }
344      }
345    }
346
347    private string GetUserName(Guid userId) {
348      try {
349        // we cannot use the ServiceLocator.Instance.UserManager since the janitor service
350        // is not hosted in the iis the MemberShip.GetUser method causes exceptions
351        // needs to be further investigated current workaround: use the authenticationcontext
352        // we could also connect to the access service to get the user name
353        using (ASPNETAuthenticationDataContext dc = new ASPNETAuthenticationDataContext()) {
354          var user = dc.aspnet_Users.SingleOrDefault(x => x.UserId == userId);
355          return user != null ? user.UserName : UnknownUserName;
356        }
357      }
358      catch (Exception) {
359        return UnknownUserName;
360      }
361    }
362
363    private class FactTaskData {
364      public int CalculationRuns { get; set; }
365      public int Retries { get; set; }
366      public long CalculatingTime { get; set; }
367      public long WaitingTime { get; set; }
368      public long TransferTime { get; set; }
369      public long InitialWaitingTime { get; set; }
370      public string Exception { get; set; }
371      public DateTime? StartTime { get; set; }
372      public DateTime? EndTime { get; set; }
373    }
374
375    private FactTaskData CalculateFactTaskData(IEnumerable<StateLog> stateLogs) {
376      var factTaskData = new FactTaskData();
377      var enumerator = stateLogs.GetEnumerator();
378      if (enumerator.MoveNext()) {
379        StateLog current = enumerator.Current, first = current, prev = null;
380        while (current != null) {
381          var next = enumerator.MoveNext() ? enumerator.Current : null;
382          int timeSpanInSeconds;
383          if (next != null) {
384            timeSpanInSeconds = (int)(next.DateTime - current.DateTime).TotalSeconds;
385          } else {
386            timeSpanInSeconds = (int)(DateTime.Now - current.DateTime).TotalSeconds;
387            factTaskData.Exception = current.Exception;
388          }
389          switch (current.State) {
390            case TaskState.Calculating:
391              factTaskData.CalculatingTime += timeSpanInSeconds;
392              factTaskData.CalculationRuns++;
393              if (factTaskData.CalculationRuns == 1) {
394                factTaskData.StartTime = current.DateTime;
395                factTaskData.InitialWaitingTime = (int)(current.DateTime - first.DateTime).TotalSeconds;
396              }
397              if (prev != null && prev.State != TaskState.Transferring) {
398                factTaskData.Retries++;
399              }
400              break;
401
402            case TaskState.Waiting:
403              factTaskData.WaitingTime += timeSpanInSeconds;
404              break;
405
406            case TaskState.Transferring:
407              factTaskData.TransferTime += timeSpanInSeconds;
408              break;
409
410            case TaskState.Finished:
411            case TaskState.Failed:
412            case TaskState.Aborted:
413              factTaskData.EndTime = current.DateTime;
414              break;
415          }
416          prev = current;
417          current = next;
418        }
419      }
420      return factTaskData;
421    }
422  }
423}
Note: See TracBrowser for help on using the repository browser.