#region License Information
/* HeuristicLab
* Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
*
* This file is part of HeuristicLab.
*
* HeuristicLab is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* HeuristicLab is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with HeuristicLab. If not, see .
*/
#endregion
using System;
using System.Collections.Generic;
using System.Data.Linq;
using System.Linq;
using HeuristicLab.Services.Access.DataAccess;
using HeuristicLab.Services.Hive.DataAccess;
using HeuristicLab.Services.Hive.DataAccess.Manager;
namespace HeuristicLab.Services.Hive {
public class HiveStatisticsGenerator : IStatisticsGenerator {
private const string UnknownUserName = "Unknown";
private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
private static readonly TaskState[] CompletedStates = { TaskState.Finished, TaskState.Aborted, TaskState.Failed };
public void GenerateStatistics() {
using (var pm = new PersistenceManager()) {
pm.UseTransaction(() => {
UpdateDimProjectTable(pm);
pm.SubmitChanges();
});
pm.UseTransaction(() => {
UpdateDimUserTable(pm);
UpdateDimJobTable(pm);
UpdateDimClientsTable(pm);
pm.SubmitChanges();
});
DimTime time = null;
pm.UseTransaction(() => {
time = UpdateDimTimeTable(pm);
pm.SubmitChanges();
});
if (time != null) {
pm.UseTransaction(() => {
UpdateFactClientInfoTable(time, pm);
pm.SubmitChanges();
UpdateFactProjectInfoTable(time, pm);
pm.SubmitChanges();
});
pm.UseTransaction(() => {
try {
UpdateFactTaskTable(pm);
UpdateExistingDimJobs(pm);
FlagJobsForDeletion(pm);
pm.SubmitChanges();
}
catch (DuplicateKeyException e) {
var logger = LogFactory.GetLogger(typeof(HiveStatisticsGenerator).Namespace);
logger.Log(string.Format(
@"Propable change from summertime to wintertime, resulting in overlapping times.
On wintertime to summertime change, slave timeouts and a fact gap will occur.
Exception Details: {0}", e));
}
});
}
}
}
private DimTime UpdateDimTimeTable(PersistenceManager pm) {
var dimTimeDao = pm.DimTimeDao;
var now = DateTime.Now;
var timeEntry = new DimTime {
Time = now,
Minute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
};
return dimTimeDao.Save(timeEntry);
}
private void UpdateDimUserTable(PersistenceManager pm) {
var dimUserDao = pm.DimUserDao;
var resourceDao = pm.ResourceDao;
var jobDao = pm.JobDao;
var existingUserIds = dimUserDao.GetAll().Select(x => x.UserId);
var vaildResourceOwnerIds = resourceDao.GetResourcesWithValidOwner().Select(x => x.OwnerUserId.Value);
var jobOwnerIds = jobDao.GetAll().Select(x => x.OwnerUserId);
var newUserIds = vaildResourceOwnerIds
.Union(jobOwnerIds)
.Where(id => !existingUserIds.Contains(id))
.ToList();
dimUserDao.Save(newUserIds.Select(x => new DimUser {
UserId = x,
Name = GetUserName(x)
}));
}
// add new projects
// delete expired projects
// update information of existing projects
private void UpdateDimProjectTable(PersistenceManager pm) {
var projectDao = pm.ProjectDao;
var dimProjectDao = pm.DimProjectDao;
var projects = projectDao.GetAll().ToList();
var dimProjects = dimProjectDao.GetAllOnlineProjects().ToList();
var onlineProjects = dimProjects.Where(x => projects.Select(y => y.ProjectId).Contains(x.ProjectId));
var addedProjects = projects.Where(x => !dimProjects.Select(y => y.ProjectId).Contains(x.ProjectId));
var removedProjects = dimProjects.Where(x => !projects.Select(y => y.ProjectId).Contains(x.ProjectId));
// set expiration time of removed projects
foreach (var p in removedProjects) {
p.DateExpired = DateTime.Now;
}
// add new projects
dimProjectDao.Save(addedProjects.Select(x => new DimProject {
ProjectId = x.ProjectId,
ParentProjectId = x.ParentProjectId,
Name = x.Name,
Description = x.Description,
OwnerUserId = x.OwnerUserId,
StartDate = x.StartDate,
EndDate = x.EndDate,
DateCreated = x.DateCreated,
DateExpired = null
}));
// expire project if its parent has changed and create a new entry
// otherwise perform "normal" update
foreach (var dimP in onlineProjects) {
var p = projects.Where(x => x.ProjectId == dimP.ProjectId).SingleOrDefault();
if (p != null) {
if (dimP.ParentProjectId == null ? p.ParentProjectId != null : dimP.ParentProjectId != p.ParentProjectId) { // or: (!object.Equals(dimP.ParentProjectId, p.ParentProjectId))
dimP.DateExpired = DateTime.Now;
dimProjectDao.Save(new DimProject {
ProjectId = p.ProjectId,
ParentProjectId = p.ParentProjectId,
Name = p.Name,
Description = p.Description,
OwnerUserId = p.OwnerUserId,
StartDate = p.StartDate,
EndDate = p.EndDate,
DateCreated = p.DateCreated,
DateExpired = null
});
} else {
dimP.Name = p.Name;
dimP.Description = p.Description;
dimP.OwnerUserId = p.OwnerUserId;
dimP.StartDate = p.StartDate;
dimP.EndDate = p.EndDate;
}
}
}
}
private void UpdateDimJobTable(PersistenceManager pm) {
var dimProjectDao = pm.DimProjectDao;
var dimJobDao = pm.DimJobDao;
var jobDao = pm.JobDao;
var taskDao = pm.TaskDao;
var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId);
var newJobs = jobDao.GetAll()
.Where(x => !dimJobIds.Contains(x.JobId))
.Select(x => new {
JobId = x.JobId,
UserId = x.OwnerUserId,
JobName = x.Name ?? string.Empty,
DateCreated = x.DateCreated,
ProjectId = dimProjectDao.GetLastValidIdByProjectId(x.ProjectId),
TotalTasks = taskDao.GetAll().Count(y => y.JobId == x.JobId)
})
.ToList();
dimJobDao.Save(newJobs.Select(x => new DimJob {
JobId = x.JobId,
JobName = x.JobName,
UserId = x.UserId,
UserName = GetUserName(x.UserId),
DateCreated = x.DateCreated,
ProjectId = x.ProjectId,
TotalTasks = x.TotalTasks,
CompletedTasks = 0,
DateCompleted = null
}));
}
private void UpdateExistingDimJobs(PersistenceManager pm) {
var dimProjectDao = pm.DimProjectDao;
var jobDao = pm.JobDao;
var dimJobDao = pm.DimJobDao;
var factTaskDao = pm.FactTaskDao;
foreach (var dimJob in dimJobDao.GetNotCompletedJobs()) {
var taskStates = factTaskDao.GetByJobId(dimJob.JobId)
.GroupBy(x => x.TaskState)
.Select(x => new {
State = x.Key,
Count = x.Count()
}).ToList();
int totalTasks = 0, completedTasks = 0;
foreach (var state in taskStates) {
totalTasks += state.Count;
if (CompletedStates.Contains(state.State)) {
completedTasks += state.Count;
}
}
var job = jobDao.GetById(dimJob.JobId);
if (totalTasks == completedTasks) {
var completeDate = factTaskDao.GetLastCompletedTaskFromJob(dimJob.JobId);
if (completeDate == null) {
if (job == null) {
completeDate = DateTime.Now;
}
}
dimJob.DateCompleted = completeDate;
}
if(job != null) {
dimJob.JobName = job.Name;
dimJob.ProjectId = dimProjectDao.GetLastValidIdByProjectId(job.ProjectId);
}
dimJob.TotalTasks = totalTasks;
dimJob.CompletedTasks = completedTasks;
}
}
private void FlagJobsForDeletion(PersistenceManager pm) {
var jobDao = pm.JobDao;
var jobs = jobDao.GetJobsReadyForDeletion();
foreach(var job in jobs) {
job.State = JobState.DeletionPending;
}
}
private void UpdateDimClientsTable(PersistenceManager pm) {
var dimClientDao = pm.DimClientDao;
var resourceDao = pm.ResourceDao;
var resources = resourceDao.GetAll().ToList();
var dimClients = dimClientDao.GetAllOnlineClients().ToList();
var onlineClients = dimClients.Where(x => resources.Select(y => y.ResourceId).Contains(x.ResourceId));
var addedResources = resources.Where(x => !dimClients.Select(y => y.ResourceId).Contains(x.ResourceId));
var removedResources = dimClients.Where(x => !resources.Select(y => y.ResourceId).Contains(x.ResourceId));
// set expiration time of removed resources
foreach(var r in removedResources) {
r.DateExpired = DateTime.Now;
}
// add new resources
dimClientDao.Save(addedResources.Select(x => new DimClient {
ResourceId = x.ResourceId,
ParentResourceId = x.ParentResourceId,
Name = x.Name,
ResourceType = x.ResourceType,
DateCreated = DateTime.Now,
DateExpired = null
}));
// expire client if its parent has changed and create a new entry
// otherwise perform "normal" update
foreach(var dimc in onlineClients) {
var r = resources.Where(x => x.ResourceId == dimc.ResourceId).SingleOrDefault();
if(r != null) {
if(dimc.ParentResourceId == null ? r.ParentResourceId != null : dimc.ParentResourceId != r.ParentResourceId) {
var now = DateTime.Now;
dimc.DateExpired = now;
dimClientDao.Save(new DimClient {
ResourceId = r.ResourceId,
ParentResourceId = r.ParentResourceId,
Name = r.Name,
ResourceType = r.ResourceType,
DateCreated = now,
DateExpired = null
});
} else {
dimc.Name = r.Name;
}
}
}
}
//// (1) for new slaves (not yet reported in Table DimClients) ...
//// and modified slaves (name or parent resource changed) a new DimClient-entry is created
//// (2) for already reported removed and modifid clients the expiration date is set
//private void UpdateDimClientsTableOld(PersistenceManager pm) {
// var dimClientDao = pm.DimClientDao;
// var slaveDao = pm.SlaveDao;
// var slaves = slaveDao.GetAll();
// var recentlyAddedClients = dimClientDao.GetAllOnlineClients();
// var slaveIds = slaves.Select(x => x.ResourceId);
// var removedClientIds = recentlyAddedClients
// .Where(x => !slaveIds.Contains(x.ResourceId))
// .Select(x => x.Id);
// var modifiedClients =
// from slave in slaves
// join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
// where (slave.Name != client.Name
// || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
// || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
// || slave.ParentResourceId != client.ResourceGroupId
// || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
// select new {
// SlaveId = slave.ResourceId,
// ClientId = client.Id
// };
// var clientIds = dimClientDao.GetAllOnlineClients().Select(x => x.ResourceId);
// var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
// var newClients = slaves
// .Where(x => !clientIds.Contains(x.ResourceId)
// || modifiedClientIds.Contains(x.ResourceId))
// .Select(x => new {
// x.ResourceId,
// x.Name,
// ResourceGroupId = x.ParentResourceId,
// GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
// ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
// GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
// })
// .ToList();
// var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
// dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
// dimClientDao.Save(newClients.Select(x => new DimClient {
// ResourceId = x.ResourceId,
// Name = x.Name,
// ExpirationTime = null,
// ResourceGroupId = x.ResourceGroupId,
// GroupName = x.GroupName,
// ResourceGroup2Id = x.ResourceGroup2Id,
// GroupName2 = x.GroupName2
// }));
//}
private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
var factClientInfoDao = pm.FactClientInfoDao;
var slaveDao = pm.SlaveDao;
var dimClientDao = pm.DimClientDao;
var newRawFactInfos =
from s in slaveDao.GetAll()
join c in dimClientDao.GetAllOnlineSlaves() on s.ResourceId equals c.ResourceId
join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
from cf in joinCf.DefaultIfEmpty()
select new {
ClientId = c.Id,
UserId = s.OwnerUserId ?? Guid.Empty,
TotalCores = s.Cores ?? 0,
FreeCores = s.FreeCores ?? 0,
TotalMemory = s.Memory ?? 0,
FreeMemory = s.FreeMemory ?? 0,
CpuUtilization = s.CpuUtilization,
SlaveState = s.SlaveState,
IsAllowedToCalculate = s.IsAllowedToCalculate,
LastFactTimestamp = cf.Timestamp
};
factClientInfoDao.Save(
from x in newRawFactInfos.ToList()
let duration = x.LastFactTimestamp != null
? (int)(newTime.Time - (DateTime)x.LastFactTimestamp).TotalSeconds
: (int)SmallestTimeSpan.TotalSeconds
select new FactClientInfo {
ClientId = x.ClientId,
DimTime = newTime,
UserId = x.UserId,
NumUsedCores = x.TotalCores - x.FreeCores,
NumTotalCores = x.TotalCores,
UsedMemory = x.TotalMemory - x.FreeMemory,
TotalMemory = x.TotalMemory,
CpuUtilization = Math.Round(x.CpuUtilization, 2),
SlaveState = x.SlaveState,
IdleTime = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0,
UnavailableTime = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0,
OfflineTime = x.SlaveState == SlaveState.Offline ? duration : 0,
IsAllowedToCalculate = x.IsAllowedToCalculate
}
);
}
private void UpdateFactProjectInfoTable(DimTime newTime, PersistenceManager pm) {
var factProjectInfoDao = pm.FactProjectInfoDao;
var dimProjectDao = pm.DimProjectDao;
var projectDao = pm.ProjectDao;
var projectAvailabilityStats = projectDao.GetAvailabilityStatsPerProject();
var projectUsageStats = projectDao.GetUsageStatsPerProject();
var dimProjects = dimProjectDao.GetAllOnlineProjects().ToList();
factProjectInfoDao.Save(
from dimp in dimProjects
let aStats = projectAvailabilityStats.Where(x => x.ProjectId == dimp.ProjectId).SingleOrDefault()
let uStats = projectUsageStats.Where(x => x.ProjectId == dimp.ProjectId).SingleOrDefault()
select new FactProjectInfo {
ProjectId = dimp.Id,
DimTime = newTime,
NumTotalCores = aStats != null ? aStats.Cores : 0,
TotalMemory = aStats != null ? aStats.Memory : 0,
NumUsedCores = uStats != null ? uStats.Cores : 0,
UsedMemory = uStats != null ? uStats.Memory : 0
}
);
}
private void UpdateFactTaskTable(PersistenceManager pm) {
var factTaskDao = pm.FactTaskDao;
var taskDao = pm.TaskDao;
var dimClientDao = pm.DimClientDao;
var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks();
//var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
// x.TaskId,
// x.LastClientId
//});
// query several properties for all new and not finished tasks
// in order to use them later either...
// (1) to update the fact task entry of not finished tasks
// (2) to insert a new fact task entry for new tasks
var newAndNotFinishedTasks =
(from task in taskDao.GetAllChildTasks()
let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
let lastSlaveId = stateLogs.First(x => x.SlaveId != null).SlaveId
where (!factTaskIds.Contains(task.TaskId)
|| notFinishedFactTasks.Select(x => x.TaskId).Contains(task.TaskId))
join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
from lastFact in lastFactPerTask.DefaultIfEmpty()
join client in dimClientDao.GetAllOnlineClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
from client in clientsPerSlaveId.DefaultIfEmpty()
select new {
TaskId = task.TaskId,
JobId = task.JobId,
Priority = task.Priority,
CoresRequired = task.CoresNeeded,
MemoryRequired = task.MemoryNeeded,
State = task.State,
StateLogs = stateLogs.OrderBy(x => x.DateTime),
LastClientId = client != null
? client.Id : lastFact != null
? lastFact.LastClientId : (Guid?)null,
NotFinishedTask = notFinishedFactTasks.Any(y => y.TaskId == task.TaskId)
}).ToList();
// (1) update data of already existing facts
// i.e. for all in newAndNotFinishedTasks where NotFinishedTask = true
foreach (var notFinishedFactTask in notFinishedFactTasks) {
var nfftUpdate = newAndNotFinishedTasks.Where(x => x.TaskId == notFinishedFactTask.TaskId).SingleOrDefault();
if(nfftUpdate != null) {
var taskData = CalculateFactTaskData(nfftUpdate.StateLogs);
notFinishedFactTask.StartTime = taskData.StartTime;
notFinishedFactTask.EndTime = taskData.EndTime;
notFinishedFactTask.LastClientId = nfftUpdate.LastClientId;
notFinishedFactTask.Priority = nfftUpdate.Priority;
notFinishedFactTask.CoresRequired = nfftUpdate.CoresRequired;
notFinishedFactTask.MemoryRequired = nfftUpdate.MemoryRequired;
notFinishedFactTask.NumCalculationRuns = taskData.CalculationRuns;
notFinishedFactTask.NumRetries = taskData.Retries;
notFinishedFactTask.WaitingTime = taskData.WaitingTime;
notFinishedFactTask.CalculatingTime = taskData.CalculatingTime;
notFinishedFactTask.TransferTime = taskData.TransferTime;
notFinishedFactTask.TaskState = nfftUpdate.State;
notFinishedFactTask.Exception = taskData.Exception;
notFinishedFactTask.InitialWaitingTime = taskData.InitialWaitingTime;
}
}
// (2) insert facts for new tasks
// i.e. for all in newAndNotFinishedTasks where NotFinishedTask = false
factTaskDao.Save(
from x in newAndNotFinishedTasks
where !x.NotFinishedTask
let taskData = CalculateFactTaskData(x.StateLogs)
select new FactTask {
TaskId = x.TaskId,
JobId = x.JobId,
StartTime = taskData.StartTime,
EndTime = taskData.EndTime,
LastClientId = x.LastClientId,
Priority = x.Priority,
CoresRequired = x.CoresRequired,
MemoryRequired = x.MemoryRequired,
NumCalculationRuns = taskData.CalculationRuns,
NumRetries = taskData.Retries,
WaitingTime = taskData.WaitingTime,
CalculatingTime = taskData.CalculatingTime,
TransferTime = taskData.TransferTime,
TaskState = x.State,
Exception = taskData.Exception,
InitialWaitingTime = taskData.InitialWaitingTime
});
////update data of already existing facts
//foreach (var notFinishedTask in factTaskDao.GetNotFinishedTasks()) {
// var ntc = newTasks.Where(x => x.TaskId == notFinishedTask.TaskId);
// if (ntc.Any()) {
// var x = ntc.Single();
// var taskData = CalculateFactTaskData(x.StateLogs);
// notFinishedTask.StartTime = taskData.StartTime;
// notFinishedTask.EndTime = taskData.EndTime;
// notFinishedTask.LastClientId = x.LastClientId;
// notFinishedTask.Priority = x.Priority;
// notFinishedTask.CoresRequired = x.CoresRequired;
// notFinishedTask.MemoryRequired = x.MemoryRequired;
// notFinishedTask.NumCalculationRuns = taskData.CalculationRuns;
// notFinishedTask.NumRetries = taskData.Retries;
// notFinishedTask.WaitingTime = taskData.WaitingTime;
// notFinishedTask.CalculatingTime = taskData.CalculatingTime;
// notFinishedTask.TransferTime = taskData.TransferTime;
// notFinishedTask.TaskState = x.State;
// notFinishedTask.Exception = taskData.Exception;
// notFinishedTask.InitialWaitingTime = taskData.InitialWaitingTime;
// }
//}
}
private string GetUserName(Guid userId) {
try {
// we cannot use the ServiceLocator.Instance.UserManager since the janitor service
// is not hosted in the iis the MemberShip.GetUser method causes exceptions
// needs to be further investigated current workaround: use the authenticationcontext
// we could also connect to the access service to get the user name
using (ASPNETAuthenticationDataContext dc = new ASPNETAuthenticationDataContext()) {
var user = dc.aspnet_Users.SingleOrDefault(x => x.UserId == userId);
return user != null ? user.UserName : UnknownUserName;
}
}
catch (Exception) {
return UnknownUserName;
}
}
private class FactTaskData {
public int CalculationRuns { get; set; }
public int Retries { get; set; }
public long CalculatingTime { get; set; }
public long WaitingTime { get; set; }
public long TransferTime { get; set; }
public long InitialWaitingTime { get; set; }
public string Exception { get; set; }
public DateTime? StartTime { get; set; }
public DateTime? EndTime { get; set; }
}
private FactTaskData CalculateFactTaskData(IEnumerable stateLogs) {
var factTaskData = new FactTaskData();
var enumerator = stateLogs.GetEnumerator();
if (enumerator.MoveNext()) {
StateLog current = enumerator.Current, first = current, prev = null;
while (current != null) {
var next = enumerator.MoveNext() ? enumerator.Current : null;
int timeSpanInSeconds;
if (next != null) {
timeSpanInSeconds = (int)(next.DateTime - current.DateTime).TotalSeconds;
} else {
timeSpanInSeconds = (int)(DateTime.Now - current.DateTime).TotalSeconds;
factTaskData.Exception = current.Exception;
}
switch (current.State) {
case TaskState.Calculating:
factTaskData.CalculatingTime += timeSpanInSeconds;
factTaskData.CalculationRuns++;
if (factTaskData.CalculationRuns == 1) {
factTaskData.StartTime = current.DateTime;
factTaskData.InitialWaitingTime = (int)(current.DateTime - first.DateTime).TotalSeconds;
}
if (prev != null && prev.State != TaskState.Transferring) {
factTaskData.Retries++;
}
break;
case TaskState.Waiting:
factTaskData.WaitingTime += timeSpanInSeconds;
break;
case TaskState.Transferring:
factTaskData.TransferTime += timeSpanInSeconds;
break;
case TaskState.Finished:
case TaskState.Failed:
case TaskState.Aborted:
factTaskData.EndTime = current.DateTime;
break;
}
prev = current;
current = next;
}
}
return factTaskData;
}
}
}