Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HiveProjectManagement/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs @ 15641

Last change on this file since 15641 was 15641, checked in by jzenisek, 6 years ago

#2839 finished implementation of job deletion routine

File size: 19.4 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Data.Linq;
25using System.Linq;
26using HeuristicLab.Services.Access.DataAccess;
27using HeuristicLab.Services.Hive.DataAccess;
28using HeuristicLab.Services.Hive.DataAccess.Manager;
29
30namespace HeuristicLab.Services.Hive {
31  public class HiveStatisticsGenerator : IStatisticsGenerator {
32
33    private const string UnknownUserName = "Unknown";
34    private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
35    private static readonly TaskState[] CompletedStates = { TaskState.Finished, TaskState.Aborted, TaskState.Failed };
36
37    public void GenerateStatistics() {
38      using (var pm = new PersistenceManager()) {
39        pm.UseTransaction(() => {
40          UpdateDimUserTable(pm);
41          UpdateDimJobTable(pm);
42          UpdateDimClientsTable(pm);
43          pm.SubmitChanges();
44        });
45
46        DimTime time = null;
47        pm.UseTransaction(() => {
48          time = UpdateDimTimeTable(pm);
49          pm.SubmitChanges();
50        });
51
52        if (time != null) {
53          pm.UseTransaction(() => {
54            UpdateFactClientInfoTable(time, pm);
55            pm.SubmitChanges();
56          });
57
58          pm.UseTransaction(() => {
59            UpdateTaskFactsTable(pm);
60            try {
61              pm.SubmitChanges();
62              UpdateExistingDimJobs(pm);
63              pm.SubmitChanges();
64            }
65            catch (DuplicateKeyException e) {
66              var logger = LogFactory.GetLogger(typeof(HiveStatisticsGenerator).Namespace);
67              logger.Log(string.Format(
68                @"Propable change from summertime to wintertime, resulting in overlapping times.
69                          On wintertime to summertime change, slave timeouts and a fact gap will occur.
70                          Exception Details: {0}", e));
71            }
72          });
73        }
74
75        pm.UseTransaction(() => {
76          FlagJobsForDeletion(pm);
77          pm.SubmitChanges();
78        });
79      }
80    }
81
82    private DimTime UpdateDimTimeTable(PersistenceManager pm) {
83      var dimTimeDao = pm.DimTimeDao;
84      var now = DateTime.Now;
85      var timeEntry = new DimTime {
86        Time = now,
87        Minute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
88        Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
89        Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
90        Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
91        Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
92      };
93      return dimTimeDao.Save(timeEntry);
94    }
95
96    private void UpdateDimUserTable(PersistenceManager pm) {
97      var dimUserDao = pm.DimUserDao;
98      var resourceDao = pm.ResourceDao;
99      var jobDao = pm.JobDao;
100      var existingUserIds = dimUserDao.GetAll().Select(x => x.UserId);
101      var vaildResourceOwnerIds = resourceDao.GetResourcesWithValidOwner().Select(x => x.OwnerUserId.Value);
102      var jobOwnerIds = jobDao.GetAll().Select(x => x.OwnerUserId);
103      var newUserIds = vaildResourceOwnerIds
104        .Union(jobOwnerIds)
105        .Where(id => !existingUserIds.Contains(id))
106        .ToList();
107      dimUserDao.Save(newUserIds.Select(x => new DimUser {
108        UserId = x,
109        Name = GetUserName(x)
110      }));
111    }
112
113    private void UpdateDimJobTable(PersistenceManager pm) {
114      var dimJobDao = pm.DimJobDao;
115      var jobDao = pm.JobDao;
116      var taskDao = pm.TaskDao;
117      var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId);
118      var newJobs = jobDao.GetAll()
119        .Where(x => !dimJobIds.Contains(x.JobId))
120        .Select(x => new {
121          JobId = x.JobId,
122          UserId = x.OwnerUserId,
123          JobName = x.Name ?? string.Empty,
124          DateCreated = x.DateCreated,
125          TotalTasks = taskDao.GetAll().Count(y => y.JobId == x.JobId)
126        })
127        .ToList();
128      dimJobDao.Save(newJobs.Select(x => new DimJob {
129        JobId = x.JobId,
130        JobName = x.JobName,
131        UserId = x.UserId,
132        UserName = GetUserName(x.UserId),
133        DateCreated = x.DateCreated,
134        TotalTasks = x.TotalTasks,
135        CompletedTasks = 0,
136        DateCompleted = null
137      }));
138    }
139
140    private void UpdateExistingDimJobs(PersistenceManager pm) {
141      var jobDao = pm.JobDao;
142      var dimJobDao = pm.DimJobDao;
143      var factTaskDao = pm.FactTaskDao;
144      foreach (var dimJob in dimJobDao.GetNotCompletedJobs()) {
145        var taskStates = factTaskDao.GetByJobId(dimJob.JobId)
146            .GroupBy(x => x.TaskState)
147            .Select(x => new {
148              State = x.Key,
149              Count = x.Count()
150            }).ToList();
151        int totalTasks = 0, completedTasks = 0;
152        foreach (var state in taskStates) {
153          totalTasks += state.Count;
154          if (CompletedStates.Contains(state.State)) {
155            completedTasks += state.Count;
156          }
157        }
158        if (totalTasks == completedTasks) {
159          var completeDate = factTaskDao.GetLastCompletedTaskFromJob(dimJob.JobId);
160          if (completeDate == null) {
161            if (jobDao.GetById(dimJob.JobId) == null) {
162              completeDate = DateTime.Now;
163            }
164          }
165          dimJob.DateCompleted = completeDate;
166        }
167        dimJob.TotalTasks = totalTasks;
168        dimJob.CompletedTasks = completedTasks;
169      }
170    }
171
172    private void FlagJobsForDeletion(PersistenceManager pm) {
173      var jobDao = pm.JobDao;
174      var jobs = jobDao.GetJobsReadyForDeletion();
175      foreach(var job in jobs) {
176        job.State = JobState.DeletionPending;
177      }
178    }
179
180    private void UpdateDimClientsTable(PersistenceManager pm) {
181      var dimClientDao = pm.DimClientDao;
182      var slaveDao = pm.SlaveDao;
183      var slaves = slaveDao.GetAll();
184      var recentlyAddedClients = dimClientDao.GetActiveClients();
185      var slaveIds = slaves.Select(x => x.ResourceId);
186
187      var removedClientIds = recentlyAddedClients
188        .Where(x => !slaveIds.Contains(x.ResourceId))
189        .Select(x => x.Id);
190      var modifiedClients =
191        from slave in slaves
192        join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
193        where (slave.Name != client.Name
194               || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
195               || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
196               || slave.ParentResourceId != client.ResourceGroupId
197               || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
198        select new {
199          SlaveId = slave.ResourceId,
200          ClientId = client.Id
201        };
202      var clientIds = dimClientDao.GetActiveClients().Select(x => x.ResourceId);
203      var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
204      var newClients = slaves
205        .Where(x => !clientIds.Contains(x.ResourceId)
206                    || modifiedClientIds.Contains(x.ResourceId))
207        .Select(x => new {
208          x.ResourceId,
209          x.Name,
210          ResourceGroupId = x.ParentResourceId,
211          GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
212          ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
213          GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
214        })
215        .ToList();
216
217      var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
218      dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
219      dimClientDao.Save(newClients.Select(x => new DimClient {
220        ResourceId = x.ResourceId,
221        Name = x.Name,
222        ExpirationTime = null,
223        ResourceGroupId = x.ResourceGroupId,
224        GroupName = x.GroupName,
225        ResourceGroup2Id = x.ResourceGroup2Id,
226        GroupName2 = x.GroupName2
227      }));
228    }
229
230    private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
231      var factClientInfoDao = pm.FactClientInfoDao;
232      var slaveDao = pm.SlaveDao;
233      var dimClientDao = pm.DimClientDao;
234
235      var newRawFactInfos =
236        from s in slaveDao.GetAll()
237        join c in dimClientDao.GetActiveClients() on s.ResourceId equals c.ResourceId
238        join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
239        from cf in joinCf.DefaultIfEmpty()
240        select new {
241          ClientId = c.Id,
242          UserId = s.OwnerUserId ?? Guid.Empty,
243          TotalCores = s.Cores ?? 0,
244          FreeCores = s.FreeCores ?? 0,
245          TotalMemory = s.Memory ?? 0,
246          FreeMemory = s.FreeMemory ?? 0,
247          CpuUtilization = s.CpuUtilization,
248          SlaveState = s.SlaveState,
249          IsAllowedToCalculate = s.IsAllowedToCalculate,
250          LastFactTimestamp = cf.Timestamp
251        };
252
253      factClientInfoDao.Save(
254        from x in newRawFactInfos.ToList()
255        let duration = x.LastFactTimestamp != null
256                       ? (int)(newTime.Time - (DateTime)x.LastFactTimestamp).TotalSeconds
257                       : (int)SmallestTimeSpan.TotalSeconds
258        select new FactClientInfo {
259          ClientId = x.ClientId,
260          DimTime = newTime,
261          UserId = x.UserId,
262          NumUsedCores = x.TotalCores - x.FreeCores,
263          NumTotalCores = x.TotalCores,
264          UsedMemory = x.TotalMemory - x.FreeMemory,
265          TotalMemory = x.TotalMemory,
266          CpuUtilization = Math.Round(x.CpuUtilization, 2),
267          SlaveState = x.SlaveState,
268          IdleTime = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0,
269          UnavailableTime = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0,
270          OfflineTime = x.SlaveState == SlaveState.Offline ? duration : 0,
271          IsAllowedToCalculate = x.IsAllowedToCalculate
272        }
273      );
274    }
275
276    private void UpdateTaskFactsTable(PersistenceManager pm) {
277      var factTaskDao = pm.FactTaskDao;
278      var taskDao = pm.TaskDao;
279      var dimClientDao = pm.DimClientDao;
280
281      var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
282      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks();
283      //var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
284      //  x.TaskId,
285      //  x.LastClientId
286      //});
287
288      // query several properties for all new and not finished tasks
289      // in order to use them later eitheir
290      // (1) to update the fact task entry of not finished tasks
291      // (2) to insert a new fact task entry for new tasks
292      var newAndNotFinishedTasks =
293        (from task in taskDao.GetAllChildTasks()
294         let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
295         let lastSlaveId = stateLogs.First(x => x.SlaveId != null).SlaveId
296         where (!factTaskIds.Contains(task.TaskId)
297                || notFinishedFactTasks.Select(x => x.TaskId).Contains(task.TaskId))
298         join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
299         from lastFact in lastFactPerTask.DefaultIfEmpty()
300         join client in dimClientDao.GetActiveClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
301         from client in clientsPerSlaveId.DefaultIfEmpty()
302         select new {
303           TaskId = task.TaskId,
304           JobId = task.JobId,
305           Priority = task.Priority,
306           CoresRequired = task.CoresNeeded,
307           MemoryRequired = task.MemoryNeeded,
308           State = task.State,
309           StateLogs = stateLogs.OrderBy(x => x.DateTime),
310           LastClientId = client != null
311                          ? client.Id : lastFact != null
312                          ? lastFact.LastClientId : (Guid?)null,
313           NotFinishedTask = notFinishedFactTasks.Any(y => y.TaskId == task.TaskId)
314         }).ToList();
315
316      // (1) update data of already existing facts
317      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = true
318      foreach (var notFinishedFactTask in notFinishedFactTasks) {
319        var nfftUpdate = newAndNotFinishedTasks.Where(x => x.TaskId == notFinishedFactTask.TaskId).SingleOrDefault();
320        if(nfftUpdate != null) {
321          var taskData = CalculateFactTaskData(nfftUpdate.StateLogs);
322
323          notFinishedFactTask.StartTime = taskData.StartTime;
324          notFinishedFactTask.EndTime = taskData.EndTime;
325          notFinishedFactTask.LastClientId = nfftUpdate.LastClientId;
326          notFinishedFactTask.Priority = nfftUpdate.Priority;
327          notFinishedFactTask.CoresRequired = nfftUpdate.CoresRequired;
328          notFinishedFactTask.MemoryRequired = nfftUpdate.MemoryRequired;
329          notFinishedFactTask.NumCalculationRuns = taskData.CalculationRuns;
330          notFinishedFactTask.NumRetries = taskData.Retries;
331          notFinishedFactTask.WaitingTime = taskData.WaitingTime;
332          notFinishedFactTask.CalculatingTime = taskData.CalculatingTime;
333          notFinishedFactTask.TransferTime = taskData.TransferTime;
334          notFinishedFactTask.TaskState = nfftUpdate.State;
335          notFinishedFactTask.Exception = taskData.Exception;
336          notFinishedFactTask.InitialWaitingTime = taskData.InitialWaitingTime;
337        }
338      }
339
340      // (2) insert facts for new tasks
341      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = false
342      factTaskDao.Save(
343        from x in newAndNotFinishedTasks
344        where !x.NotFinishedTask
345        let taskData = CalculateFactTaskData(x.StateLogs)
346        select new FactTask {
347          TaskId = x.TaskId,
348          JobId = x.JobId,
349          StartTime = taskData.StartTime,
350          EndTime = taskData.EndTime,
351          LastClientId = x.LastClientId,
352          Priority = x.Priority,
353          CoresRequired = x.CoresRequired,
354          MemoryRequired = x.MemoryRequired,
355          NumCalculationRuns = taskData.CalculationRuns,
356          NumRetries = taskData.Retries,
357          WaitingTime = taskData.WaitingTime,
358          CalculatingTime = taskData.CalculatingTime,
359          TransferTime = taskData.TransferTime,
360          TaskState = x.State,
361          Exception = taskData.Exception,
362          InitialWaitingTime = taskData.InitialWaitingTime
363        });
364
365
366      ////update data of already existing facts
367      //foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
368      //  var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
369      //  if (ntc.Any()) {
370      //    var x = ntc.Single();
371      //    var taskData = CalculateFactTaskData(x.StateLogs);
372
373      //    notFinishedTask.StartTime = taskData.StartTime;
374      //    notFinishedTask.EndTime = taskData.EndTime;
375      //    notFinishedTask.LastClientId = x.LastClientId;
376      //    notFinishedTask.Priority = x.Priority;
377      //    notFinishedTask.CoresRequired = x.CoresRequired;
378      //    notFinishedTask.MemoryRequired = x.MemoryRequired;
379      //    notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
380      //    notFinishedTask.NumRetries = taskData.Retries;
381      //    notFinishedTask.WaitingTime = taskData.WaitingTime;
382      //    notFinishedTask.CalculatingTime = taskData.CalculatingTime;
383      //    notFinishedTask.TransferTime = taskData.TransferTime;
384      //    notFinishedTask.TaskState = x.State;
385      //    notFinishedTask.Exception = taskData.Exception;
386      //    notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
387      //  }
388      //}
389    }
390
391    private string GetUserName(Guid userId) {
392      try {
393        // we cannot use the ServiceLocator.Instance.UserManager since the janitor service
394        // is not hosted in the iis the MemberShip.GetUser method causes exceptions
395        // needs to be further investigated current workaround: use the authenticationcontext
396        // we could also connect to the access service to get the user name
397        using (ASPNETAuthenticationDataContext dc = new ASPNETAuthenticationDataContext()) {
398          var user = dc.aspnet_Users.SingleOrDefault(x => x.UserId == userId);
399          return user != null ? user.UserName : UnknownUserName;
400        }
401      }
402      catch (Exception) {
403        return UnknownUserName;
404      }
405    }
406
407    private class FactTaskData {
408      public int CalculationRuns { get; set; }
409      public int Retries { get; set; }
410      public long CalculatingTime { get; set; }
411      public long WaitingTime { get; set; }
412      public long TransferTime { get; set; }
413      public long InitialWaitingTime { get; set; }
414      public string Exception { get; set; }
415      public DateTime? StartTime { get; set; }
416      public DateTime? EndTime { get; set; }
417    }
418
419    private FactTaskData CalculateFactTaskData(IEnumerable<StateLog> stateLogs) {
420      var factTaskData = new FactTaskData();
421      var enumerator = stateLogs.GetEnumerator();
422      if (enumerator.MoveNext()) {
423        StateLog current = enumerator.Current, first = current, prev = null;
424        while (current != null) {
425          var next = enumerator.MoveNext() ? enumerator.Current : null;
426          int timeSpanInSeconds;
427          if (next != null) {
428            timeSpanInSeconds = (int)(next.DateTime - current.DateTime).TotalSeconds;
429          } else {
430            timeSpanInSeconds = (int)(DateTime.Now - current.DateTime).TotalSeconds;
431            factTaskData.Exception = current.Exception;
432          }
433          switch (current.State) {
434            case TaskState.Calculating:
435              factTaskData.CalculatingTime += timeSpanInSeconds;
436              factTaskData.CalculationRuns++;
437              if (factTaskData.CalculationRuns == 1) {
438                factTaskData.StartTime = current.DateTime;
439                factTaskData.InitialWaitingTime = (int)(current.DateTime - first.DateTime).TotalSeconds;
440              }
441              if (prev != null && prev.State != TaskState.Transferring) {
442                factTaskData.Retries++;
443              }
444              break;
445
446            case TaskState.Waiting:
447              factTaskData.WaitingTime += timeSpanInSeconds;
448              break;
449
450            case TaskState.Transferring:
451              factTaskData.TransferTime += timeSpanInSeconds;
452              break;
453
454            case TaskState.Finished:
455            case TaskState.Failed:
456            case TaskState.Aborted:
457              factTaskData.EndTime = current.DateTime;
458              break;
459          }
460          prev = current;
461          current = next;
462        }
463      }
464      return factTaskData;
465    }
466  }
467}
Note: See TracBrowser for help on using the repository browser.