source: branches/HiveProjectManagement/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs @ 15659

Last change on this file since 15659 was 15659, checked in by jzenisek, 3 years ago

#2839

  • added DimProject and FactProjectInfo entities to statistics generation
  • implemented tracking for projects
File size: 22.6 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Data.Linq;
25using System.Linq;
26using HeuristicLab.Services.Access.DataAccess;
27using HeuristicLab.Services.Hive.DataAccess;
28using HeuristicLab.Services.Hive.DataAccess.Manager;
29
30namespace HeuristicLab.Services.Hive {
31  public class HiveStatisticsGenerator : IStatisticsGenerator {
32
33    private const string UnknownUserName = "Unknown";
34    private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
35    private static readonly TaskState[] CompletedStates = { TaskState.Finished, TaskState.Aborted, TaskState.Failed };
36
37    public void GenerateStatistics() {
38      using (var pm = new PersistenceManager()) {
39
40        pm.UseTransaction(() => {
41          UpdateDimProjectTable(pm);
42        });
43
44        pm.UseTransaction(() => {
45          UpdateDimUserTable(pm);
46         
47          UpdateDimJobTable(pm);
48          UpdateDimClientsTable(pm);
49          pm.SubmitChanges();
50        });
51
52        DimTime time = null;
53        pm.UseTransaction(() => {
54          time = UpdateDimTimeTable(pm);
55          pm.SubmitChanges();
56        });
57
58        if (time != null) {
59          pm.UseTransaction(() => {
60            UpdateFactClientInfoTable(time, pm);
61            UpdateFactProjectInfoTable(time, pm); // in progress
62            pm.SubmitChanges();
63          });
64
65          pm.UseTransaction(() => {
66            UpdateTaskFactsTable(pm);
67            try {
68              pm.SubmitChanges();
69              UpdateExistingDimJobs(pm);
70              pm.SubmitChanges();
71            }
72            catch (DuplicateKeyException e) {
73              var logger = LogFactory.GetLogger(typeof(HiveStatisticsGenerator).Namespace);
74              logger.Log(string.Format(
75                @"Propable change from summertime to wintertime, resulting in overlapping times.
76                          On wintertime to summertime change, slave timeouts and a fact gap will occur.
77                          Exception Details: {0}", e));
78            }
79          });
80        }
81
82        pm.UseTransaction(() => {
83          FlagJobsForDeletion(pm);
84          pm.SubmitChanges();
85        });
86      }
87    }
88
89    private DimTime UpdateDimTimeTable(PersistenceManager pm) {
90      var dimTimeDao = pm.DimTimeDao;
91      var now = DateTime.Now;
92      var timeEntry = new DimTime {
93        Time = now,
94        Minute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
95        Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
96        Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
97        Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
98        Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
99      };
100      return dimTimeDao.Save(timeEntry);
101    }
102
103    private void UpdateDimUserTable(PersistenceManager pm) {
104      var dimUserDao = pm.DimUserDao;
105      var resourceDao = pm.ResourceDao;
106      var jobDao = pm.JobDao;
107      var existingUserIds = dimUserDao.GetAll().Select(x => x.UserId);
108      var vaildResourceOwnerIds = resourceDao.GetResourcesWithValidOwner().Select(x => x.OwnerUserId.Value);
109      var jobOwnerIds = jobDao.GetAll().Select(x => x.OwnerUserId);
110      var newUserIds = vaildResourceOwnerIds
111        .Union(jobOwnerIds)
112        .Where(id => !existingUserIds.Contains(id))
113        .ToList();
114      dimUserDao.Save(newUserIds.Select(x => new DimUser {
115        UserId = x,
116        Name = GetUserName(x)
117      }));
118    }
119
120    private void UpdateDimProjectTable(PersistenceManager pm) {
121      var projectDao = pm.ProjectDao;
122      var dimProjectDao = pm.DimProjectDao;
123
124      var projects = projectDao.GetAll().ToList();
125      var dimProjects = dimProjectDao.GetAllOnlineProjects().ToList();
126
127      var onlineProjects = dimProjects.Where(x => projects.Select(y => y.ProjectId).Contains(x.ProjectId));
128      var addedProjects = projects.Where(x => !dimProjects.Select(y => y.ProjectId).Contains(x.ProjectId));
129      var removedProjects = dimProjects.Where(x => !projects.Select(y => y.ProjectId).Contains(x.ProjectId));
130
131      // set expiration time of removed projects
132      foreach (var p in removedProjects) {
133        p.DateExpired = DateTime.Now;
134      }
135
136      // add new projects
137      dimProjectDao.Save(addedProjects.Select(x => new DimProject {
138        ProjectId = x.ProjectId,
139        ParentProjectId = x.ParentProjectId,
140        Name = x.Name,
141        Description = x.Description,
142        OwnerUserId = x.OwnerUserId,
143        StartDate = x.StartDate,
144        EndDate = x.EndDate,
145        DateCreated = x.DateCreated,
146        DateExpired = null
147      }));
148
149      // if a project's parent has changed expire entry in DimProject and create a new entry
150      // else perform "normal" update
151      foreach (var dimP in onlineProjects) {
152        var p = projects.Where(x => x.ProjectId == dimP.ProjectId).SingleOrDefault();
153        if (p != null) {
154          if (dimP.ParentProjectId != p.ParentProjectId) {
155            dimP.DateExpired = DateTime.Now;
156            dimProjectDao.Save(new DimProject {
157              ProjectId = p.ProjectId,
158              ParentProjectId = p.ParentProjectId,
159              Name = p.Name,
160              Description = p.Description,
161              OwnerUserId = p.OwnerUserId,
162              StartDate = p.StartDate,
163              EndDate = p.EndDate,
164              DateCreated = p.DateCreated,
165              DateExpired = null
166            });
167          } else {
168            dimP.Name = p.Name;
169            dimP.Description = p.Description;
170            dimP.OwnerUserId = p.OwnerUserId;
171            dimP.StartDate = p.StartDate;
172            dimP.EndDate = p.EndDate;
173          }
174        }
175      }
176    }
177
178    private void UpdateDimJobTable(PersistenceManager pm) {
179      var dimProjectDao = pm.DimProjectDao;
180      var dimJobDao = pm.DimJobDao;
181      var jobDao = pm.JobDao;
182      var taskDao = pm.TaskDao;
183      var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId);
184      var newJobs = jobDao.GetAll()
185        .Where(x => !dimJobIds.Contains(x.JobId))
186        .Select(x => new {
187          JobId = x.JobId,
188          UserId = x.OwnerUserId,
189          JobName = x.Name ?? string.Empty,
190          DateCreated = x.DateCreated,
191          ProjectId = dimProjectDao.GetLastValidIdByProjectId(x.ProjectId),
192          TotalTasks = taskDao.GetAll().Count(y => y.JobId == x.JobId)
193        })
194        .ToList();
195      dimJobDao.Save(newJobs.Select(x => new DimJob {
196        JobId = x.JobId,
197        JobName = x.JobName,
198        UserId = x.UserId,
199        UserName = GetUserName(x.UserId),
200        DateCreated = x.DateCreated,
201        ProjectId = x.ProjectId,
202        TotalTasks = x.TotalTasks,
203        CompletedTasks = 0,
204        DateCompleted = null
205      }));
206    }
207
208    private void UpdateExistingDimJobs(PersistenceManager pm) {
209      var dimProjectDao = pm.DimProjectDao;
210      var jobDao = pm.JobDao;
211      var dimJobDao = pm.DimJobDao;
212      var factTaskDao = pm.FactTaskDao;
213      foreach (var dimJob in dimJobDao.GetNotCompletedJobs()) {
214        var taskStates = factTaskDao.GetByJobId(dimJob.JobId)
215            .GroupBy(x => x.TaskState)
216            .Select(x => new {
217              State = x.Key,
218              Count = x.Count()
219            }).ToList();
220        int totalTasks = 0, completedTasks = 0;
221        foreach (var state in taskStates) {
222          totalTasks += state.Count;
223          if (CompletedStates.Contains(state.State)) {
224            completedTasks += state.Count;
225          }
226        }
227        var job = jobDao.GetById(dimJob.JobId);
228        if (totalTasks == completedTasks) {
229          var completeDate = factTaskDao.GetLastCompletedTaskFromJob(dimJob.JobId);
230          if (completeDate == null) {
231            if (job == null) {
232              completeDate = DateTime.Now;
233            }
234          }
235          dimJob.DateCompleted = completeDate;
236        }
237        if(job != null) {
238          dimJob.JobName = job.Name;
239          dimJob.ProjectId = dimProjectDao.GetLastValidIdByProjectId(job.ProjectId);
240        }
241
242        dimJob.TotalTasks = totalTasks;
243        dimJob.CompletedTasks = completedTasks;
244      }
245    }
246
247    private void FlagJobsForDeletion(PersistenceManager pm) {
248      var jobDao = pm.JobDao;
249      var jobs = jobDao.GetJobsReadyForDeletion();
250      foreach(var job in jobs) {
251        job.State = JobState.DeletionPending;
252      }
253    }
254
255    // (1) for new slaves (not yet reported in Table DimClients) ...
256    // and modified slaves (name or parent resource changed) a new DimClient-entry is created
257    // (2) for already reported removed and modifid clients the expiration date is set
258    private void UpdateDimClientsTable(PersistenceManager pm) {
259      var dimClientDao = pm.DimClientDao;
260      var slaveDao = pm.SlaveDao;
261      var slaves = slaveDao.GetAll();
262      var recentlyAddedClients = dimClientDao.GetActiveClients();
263      var slaveIds = slaves.Select(x => x.ResourceId);
264
265      var removedClientIds = recentlyAddedClients
266        .Where(x => !slaveIds.Contains(x.ResourceId))
267        .Select(x => x.Id);
268      var modifiedClients =
269        from slave in slaves
270        join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
271        where (slave.Name != client.Name
272               || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
273               || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
274               || slave.ParentResourceId != client.ResourceGroupId
275               || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
276        select new {
277          SlaveId = slave.ResourceId,
278          ClientId = client.Id
279        };
280      var clientIds = dimClientDao.GetActiveClients().Select(x => x.ResourceId);
281      var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
282      var newClients = slaves
283        .Where(x => !clientIds.Contains(x.ResourceId)
284                    || modifiedClientIds.Contains(x.ResourceId))
285        .Select(x => new {
286          x.ResourceId,
287          x.Name,
288          ResourceGroupId = x.ParentResourceId,
289          GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
290          ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
291          GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
292        })
293        .ToList();
294
295      var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
296      dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
297      dimClientDao.Save(newClients.Select(x => new DimClient {
298        ResourceId = x.ResourceId,
299        Name = x.Name,
300        ExpirationTime = null,
301        ResourceGroupId = x.ResourceGroupId,
302        GroupName = x.GroupName,
303        ResourceGroup2Id = x.ResourceGroup2Id,
304        GroupName2 = x.GroupName2
305      }));
306    }
307
308    private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
309      var factClientInfoDao = pm.FactClientInfoDao;
310      var slaveDao = pm.SlaveDao;
311      var dimClientDao = pm.DimClientDao;
312
313      var newRawFactInfos =
314        from s in slaveDao.GetAll()
315        join c in dimClientDao.GetActiveClients() on s.ResourceId equals c.ResourceId
316        join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
317        from cf in joinCf.DefaultIfEmpty()
318        select new {
319          ClientId = c.Id,
320          UserId = s.OwnerUserId ?? Guid.Empty,
321          TotalCores = s.Cores ?? 0,
322          FreeCores = s.FreeCores ?? 0,
323          TotalMemory = s.Memory ?? 0,
324          FreeMemory = s.FreeMemory ?? 0,
325          CpuUtilization = s.CpuUtilization,
326          SlaveState = s.SlaveState,
327          IsAllowedToCalculate = s.IsAllowedToCalculate,
328          LastFactTimestamp = cf.Timestamp
329        };
330
331      factClientInfoDao.Save(
332        from x in newRawFactInfos.ToList()
333        let duration = x.LastFactTimestamp != null
334                       ? (int)(newTime.Time - (DateTime)x.LastFactTimestamp).TotalSeconds
335                       : (int)SmallestTimeSpan.TotalSeconds
336        select new FactClientInfo {
337          ClientId = x.ClientId,
338          DimTime = newTime,
339          UserId = x.UserId,
340          NumUsedCores = x.TotalCores - x.FreeCores,
341          NumTotalCores = x.TotalCores,
342          UsedMemory = x.TotalMemory - x.FreeMemory,
343          TotalMemory = x.TotalMemory,
344          CpuUtilization = Math.Round(x.CpuUtilization, 2),
345          SlaveState = x.SlaveState,
346          IdleTime = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0,
347          UnavailableTime = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0,
348          OfflineTime = x.SlaveState == SlaveState.Offline ? duration : 0,
349          IsAllowedToCalculate = x.IsAllowedToCalculate
350        }
351      );
352    }
353
354    private void UpdateFactProjectInfoTable(DimTime newTime, PersistenceManager pm) {
355      // TODO
356    }
357
358    private void UpdateTaskFactsTable(PersistenceManager pm) {
359      var factTaskDao = pm.FactTaskDao;
360      var taskDao = pm.TaskDao;
361      var dimClientDao = pm.DimClientDao;
362
363      var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
364      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks();
365      //var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
366      //  x.TaskId,
367      //  x.LastClientId
368      //});
369
370      // query several properties for all new and not finished tasks
371      // in order to use them later eitheir
372      // (1) to update the fact task entry of not finished tasks
373      // (2) to insert a new fact task entry for new tasks
374      var newAndNotFinishedTasks =
375        (from task in taskDao.GetAllChildTasks()
376         let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
377         let lastSlaveId = stateLogs.First(x => x.SlaveId != null).SlaveId
378         where (!factTaskIds.Contains(task.TaskId)
379                || notFinishedFactTasks.Select(x => x.TaskId).Contains(task.TaskId))
380         join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
381         from lastFact in lastFactPerTask.DefaultIfEmpty()
382         join client in dimClientDao.GetActiveClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
383         from client in clientsPerSlaveId.DefaultIfEmpty()
384         select new {
385           TaskId = task.TaskId,
386           JobId = task.JobId,
387           Priority = task.Priority,
388           CoresRequired = task.CoresNeeded,
389           MemoryRequired = task.MemoryNeeded,
390           State = task.State,
391           StateLogs = stateLogs.OrderBy(x => x.DateTime),
392           LastClientId = client != null
393                          ? client.Id : lastFact != null
394                          ? lastFact.LastClientId : (Guid?)null,
395           NotFinishedTask = notFinishedFactTasks.Any(y => y.TaskId == task.TaskId)
396         }).ToList();
397
398      // (1) update data of already existing facts
399      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = true
400      foreach (var notFinishedFactTask in notFinishedFactTasks) {
401        var nfftUpdate = newAndNotFinishedTasks.Where(x => x.TaskId == notFinishedFactTask.TaskId).SingleOrDefault();
402        if(nfftUpdate != null) {
403          var taskData = CalculateFactTaskData(nfftUpdate.StateLogs);
404
405          notFinishedFactTask.StartTime = taskData.StartTime;
406          notFinishedFactTask.EndTime = taskData.EndTime;
407          notFinishedFactTask.LastClientId = nfftUpdate.LastClientId;
408          notFinishedFactTask.Priority = nfftUpdate.Priority;
409          notFinishedFactTask.CoresRequired = nfftUpdate.CoresRequired;
410          notFinishedFactTask.MemoryRequired = nfftUpdate.MemoryRequired;
411          notFinishedFactTask.NumCalculationRuns = taskData.CalculationRuns;
412          notFinishedFactTask.NumRetries = taskData.Retries;
413          notFinishedFactTask.WaitingTime = taskData.WaitingTime;
414          notFinishedFactTask.CalculatingTime = taskData.CalculatingTime;
415          notFinishedFactTask.TransferTime = taskData.TransferTime;
416          notFinishedFactTask.TaskState = nfftUpdate.State;
417          notFinishedFactTask.Exception = taskData.Exception;
418          notFinishedFactTask.InitialWaitingTime = taskData.InitialWaitingTime;
419        }
420      }
421
422      // (2) insert facts for new tasks
423      // i.e. for all in newAndNotFinishedTasks where NotFinishedTask = false
424      factTaskDao.Save(
425        from x in newAndNotFinishedTasks
426        where !x.NotFinishedTask
427        let taskData = CalculateFactTaskData(x.StateLogs)
428        select new FactTask {
429          TaskId = x.TaskId,
430          JobId = x.JobId,
431          StartTime = taskData.StartTime,
432          EndTime = taskData.EndTime,
433          LastClientId = x.LastClientId,
434          Priority = x.Priority,
435          CoresRequired = x.CoresRequired,
436          MemoryRequired = x.MemoryRequired,
437          NumCalculationRuns = taskData.CalculationRuns,
438          NumRetries = taskData.Retries,
439          WaitingTime = taskData.WaitingTime,
440          CalculatingTime = taskData.CalculatingTime,
441          TransferTime = taskData.TransferTime,
442          TaskState = x.State,
443          Exception = taskData.Exception,
444          InitialWaitingTime = taskData.InitialWaitingTime
445        });
446
447
448      ////update data of already existing facts
449      //foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
450      //  var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
451      //  if (ntc.Any()) {
452      //    var x = ntc.Single();
453      //    var taskData = CalculateFactTaskData(x.StateLogs);
454
455      //    notFinishedTask.StartTime = taskData.StartTime;
456      //    notFinishedTask.EndTime = taskData.EndTime;
457      //    notFinishedTask.LastClientId = x.LastClientId;
458      //    notFinishedTask.Priority = x.Priority;
459      //    notFinishedTask.CoresRequired = x.CoresRequired;
460      //    notFinishedTask.MemoryRequired = x.MemoryRequired;
461      //    notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
462      //    notFinishedTask.NumRetries = taskData.Retries;
463      //    notFinishedTask.WaitingTime = taskData.WaitingTime;
464      //    notFinishedTask.CalculatingTime = taskData.CalculatingTime;
465      //    notFinishedTask.TransferTime = taskData.TransferTime;
466      //    notFinishedTask.TaskState = x.State;
467      //    notFinishedTask.Exception = taskData.Exception;
468      //    notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
469      //  }
470      //}
471    }
472
473    private string GetUserName(Guid userId) {
474      try {
475        // we cannot use the ServiceLocator.Instance.UserManager since the janitor service
476        // is not hosted in the iis the MemberShip.GetUser method causes exceptions
477        // needs to be further investigated current workaround: use the authenticationcontext
478        // we could also connect to the access service to get the user name
479        using (ASPNETAuthenticationDataContext dc = new ASPNETAuthenticationDataContext()) {
480          var user = dc.aspnet_Users.SingleOrDefault(x => x.UserId == userId);
481          return user != null ? user.UserName : UnknownUserName;
482        }
483      }
484      catch (Exception) {
485        return UnknownUserName;
486      }
487    }
488
489    private class FactTaskData {
490      public int CalculationRuns { get; set; }
491      public int Retries { get; set; }
492      public long CalculatingTime { get; set; }
493      public long WaitingTime { get; set; }
494      public long TransferTime { get; set; }
495      public long InitialWaitingTime { get; set; }
496      public string Exception { get; set; }
497      public DateTime? StartTime { get; set; }
498      public DateTime? EndTime { get; set; }
499    }
500
501    private FactTaskData CalculateFactTaskData(IEnumerable<StateLog> stateLogs) {
502      var factTaskData = new FactTaskData();
503      var enumerator = stateLogs.GetEnumerator();
504      if (enumerator.MoveNext()) {
505        StateLog current = enumerator.Current, first = current, prev = null;
506        while (current != null) {
507          var next = enumerator.MoveNext() ? enumerator.Current : null;
508          int timeSpanInSeconds;
509          if (next != null) {
510            timeSpanInSeconds = (int)(next.DateTime - current.DateTime).TotalSeconds;
511          } else {
512            timeSpanInSeconds = (int)(DateTime.Now - current.DateTime).TotalSeconds;
513            factTaskData.Exception = current.Exception;
514          }
515          switch (current.State) {
516            case TaskState.Calculating:
517              factTaskData.CalculatingTime += timeSpanInSeconds;
518              factTaskData.CalculationRuns++;
519              if (factTaskData.CalculationRuns == 1) {
520                factTaskData.StartTime = current.DateTime;
521                factTaskData.InitialWaitingTime = (int)(current.DateTime - first.DateTime).TotalSeconds;
522              }
523              if (prev != null && prev.State != TaskState.Transferring) {
524                factTaskData.Retries++;
525              }
526              break;
527
528            case TaskState.Waiting:
529              factTaskData.WaitingTime += timeSpanInSeconds;
530              break;
531
532            case TaskState.Transferring:
533              factTaskData.TransferTime += timeSpanInSeconds;
534              break;
535
536            case TaskState.Finished:
537            case TaskState.Failed:
538            case TaskState.Aborted:
539              factTaskData.EndTime = current.DateTime;
540              break;
541          }
542          prev = current;
543          current = next;
544        }
545      }
546      return factTaskData;
547    }
548  }
549}
Note: See TracBrowser for help on using the repository browser.