Changeset 15716


Ignore:
Timestamp:
02/02/18 20:21:28 (19 months ago)
Author:
jzenisek
Message:

#2839 worked on stats generation

File:
1 edited

Legend:

Unmodified
Added
Removed
  • branches/HiveProjectManagement/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

    r15666 r15716  
    4040        pm.UseTransaction(() => {
    4141          UpdateDimProjectTable(pm);
     42          pm.SubmitChanges();
    4243        });
    4344
     
    5960          pm.UseTransaction(() => {
    6061            UpdateFactClientInfoTable(time, pm);
     62            pm.SubmitChanges();
    6163            UpdateFactProjectInfoTable(time, pm);
    6264            pm.SubmitChanges();
     
    6466
    6567          pm.UseTransaction(() => {
    66             UpdateTaskFactsTable(pm);
     68            UpdateFactTaskTable(pm);
    6769            try {
    6870              pm.SubmitChanges();
     
    150152      }));
    151153
    152       // if a project's parent has changed expire entry in DimProject and create a new entry
    153       // else perform "normal" update
     154      // expire project if its parent has changed and create a new entry
     155      // otherwise perform "normal" update
    154156      foreach (var dimP in onlineProjects) {
    155157        var p = projects.Where(x => x.ProjectId == dimP.ProjectId).SingleOrDefault();
    156158        if (p != null) {
    157           if (dimP.ParentProjectId != p.ParentProjectId) {
     159          if (dimP.ParentProjectId == null ? p.ParentProjectId != null : dimP.ParentProjectId != p.ParentProjectId) { // or: (!object.Equals(dimP.ParentProjectId, p.ParentProjectId))
    158160            dimP.DateExpired = DateTime.Now;
    159161            dimProjectDao.Save(new DimProject {
     
    256258    }
    257259
    258     // (1) for new slaves (not yet reported in Table DimClients) ...
    259     // and modified slaves (name or parent resource changed) a new DimClient-entry is created
    260     // (2) for already reported removed and modifid clients the expiration date is set
    261260    private void UpdateDimClientsTable(PersistenceManager pm) {
    262261      var dimClientDao = pm.DimClientDao;
    263       var slaveDao = pm.SlaveDao;
    264       var slaves = slaveDao.GetAll();
    265       var recentlyAddedClients = dimClientDao.GetActiveClients();
    266       var slaveIds = slaves.Select(x => x.ResourceId);
    267 
    268       var removedClientIds = recentlyAddedClients
    269         .Where(x => !slaveIds.Contains(x.ResourceId))
    270         .Select(x => x.Id);
    271       var modifiedClients =
    272         from slave in slaves
    273         join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
    274         where (slave.Name != client.Name
    275                || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
    276                || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
    277                || slave.ParentResourceId != client.ResourceGroupId
    278                || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
    279         select new {
    280           SlaveId = slave.ResourceId,
    281           ClientId = client.Id
    282         };
    283       var clientIds = dimClientDao.GetActiveClients().Select(x => x.ResourceId);
    284       var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
    285       var newClients = slaves
    286         .Where(x => !clientIds.Contains(x.ResourceId)
    287                     || modifiedClientIds.Contains(x.ResourceId))
    288         .Select(x => new {
    289           x.ResourceId,
    290           x.Name,
    291           ResourceGroupId = x.ParentResourceId,
    292           GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
    293           ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
    294           GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
    295         })
    296         .ToList();
    297 
    298       var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
    299       dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
    300       dimClientDao.Save(newClients.Select(x => new DimClient {
     262      var resourceDao = pm.ResourceDao;
     263
     264      var resources = resourceDao.GetAll().ToList();
     265      var dimClients = dimClientDao.GetAllOnlineClients().ToList();
     266
     267      var onlineClients = dimClients.Where(x => resources.Select(y => y.ResourceId).Contains(x.ResourceId));
     268      var addedResources = resources.Where(x => !dimClients.Select(y => y.ResourceId).Contains(x.ResourceId));
     269      var removedResources = dimClients.Where(x => !resources.Select(y => y.ResourceId).Contains(x.ResourceId));
     270
     271      // set expiration time of removed resources
     272      foreach(var r in removedResources) {
     273        r.DateExpired = DateTime.Now;
     274      }
     275
     276      // add new resources
     277      dimClientDao.Save(addedResources.Select(x => new DimClient {
    301278        ResourceId = x.ResourceId,
     279        ParentResourceId = x.ParentResourceId,
    302280        Name = x.Name,
    303         ExpirationTime = null,
    304         ResourceGroupId = x.ResourceGroupId,
    305         GroupName = x.GroupName,
    306         ResourceGroup2Id = x.ResourceGroup2Id,
    307         GroupName2 = x.GroupName2
     281        ResourceType = x.ResourceType,
     282        DateCreated = DateTime.Now,
     283        DateExpired = null
    308284      }));
    309     }
     285
     286      // expire client if its parent has changed and create a new entry
     287      // otherwise perform "normal" update
     288      foreach(var dimc in onlineClients) {
     289        var r = resources.Where(x => x.ResourceId == dimc.ResourceId).SingleOrDefault();
     290        if(r != null) {
     291          if(dimc.ParentResourceId == null ? r.ParentResourceId != null : dimc.ParentResourceId != r.ParentResourceId) {
     292            var now = DateTime.Now;
     293            dimc.DateExpired = now;
     294            dimClientDao.Save(new DimClient {
     295              ResourceId = r.ResourceId,
     296              ParentResourceId = r.ParentResourceId,
     297              Name = r.Name,
     298              ResourceType = r.ResourceType,
     299              DateCreated = now,
     300              DateExpired = null
     301            });
     302          } else {
     303            dimc.Name = r.Name;
     304          }
     305        }
     306      }
     307    }
     308
     309    //// (1) for new slaves (not yet reported in Table DimClients) ...
     310    //// and modified slaves (name or parent resource changed) a new DimClient-entry is created
     311    //// (2) for already reported removed and modifid clients the expiration date is set
     312    //private void UpdateDimClientsTableOld(PersistenceManager pm) {
     313    //  var dimClientDao = pm.DimClientDao;
     314    //  var slaveDao = pm.SlaveDao;
     315    //  var slaves = slaveDao.GetAll();
     316    //  var recentlyAddedClients = dimClientDao.GetAllOnlineClients();
     317    //  var slaveIds = slaves.Select(x => x.ResourceId);
     318
     319    //  var removedClientIds = recentlyAddedClients
     320    //    .Where(x => !slaveIds.Contains(x.ResourceId))
     321    //    .Select(x => x.Id);
     322    //  var modifiedClients =
     323    //    from slave in slaves
     324    //    join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
     325    //    where (slave.Name != client.Name
     326    //           || slave.ParentResourceId == null && client.ResourceGroupId != null // because both can be null and null comparison
     327    //           || slave.ParentResourceId != null && client.ResourceGroupId == null // does return no entry on the sql server
     328    //           || slave.ParentResourceId != client.ResourceGroupId
     329    //           || ((slave.ParentResource != null) && slave.ParentResource.ParentResourceId != client.ResourceGroup2Id))
     330    //    select new {
     331    //      SlaveId = slave.ResourceId,
     332    //      ClientId = client.Id
     333    //    };
     334    //  var clientIds = dimClientDao.GetAllOnlineClients().Select(x => x.ResourceId);
     335    //  var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
     336    //  var newClients = slaves
     337    //    .Where(x => !clientIds.Contains(x.ResourceId)
     338    //                || modifiedClientIds.Contains(x.ResourceId))
     339    //    .Select(x => new {
     340    //      x.ResourceId,
     341    //      x.Name,
     342    //      ResourceGroupId = x.ParentResourceId,
     343    //      GroupName = x.ParentResource != null ? x.ParentResource.Name : null,
     344    //      ResourceGroup2Id = x.ParentResource != null ? x.ParentResource.ParentResourceId : null,
     345    //      GroupName2 = x.ParentResource != null ? x.ParentResource.ParentResource != null ? x.ParentResource.ParentResource.Name : null : null
     346    //    })
     347    //    .ToList();
     348
     349    //  var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
     350    //  dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
     351    //  dimClientDao.Save(newClients.Select(x => new DimClient {
     352    //    ResourceId = x.ResourceId,
     353    //    Name = x.Name,
     354    //    ExpirationTime = null,
     355    //    ResourceGroupId = x.ResourceGroupId,
     356    //    GroupName = x.GroupName,
     357    //    ResourceGroup2Id = x.ResourceGroup2Id,
     358    //    GroupName2 = x.GroupName2
     359    //  }));
     360    //}
     361
    310362
    311363    private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
     
    316368      var newRawFactInfos =
    317369        from s in slaveDao.GetAll()
    318         join c in dimClientDao.GetActiveClients() on s.ResourceId equals c.ResourceId
     370        join c in dimClientDao.GetAllOnlineSlaves() on s.ResourceId equals c.ResourceId
    319371        join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
    320372        from cf in joinCf.DefaultIfEmpty()
     
    379431    }
    380432
    381     private void UpdateTaskFactsTable(PersistenceManager pm) {
     433    private void UpdateFactTaskTable(PersistenceManager pm) {
    382434      var factTaskDao = pm.FactTaskDao;
    383435      var taskDao = pm.TaskDao;
     
    392444
    393445      // query several properties for all new and not finished tasks
    394       // in order to use them later eitheir
     446      // in order to use them later either...
    395447      // (1) to update the fact task entry of not finished tasks
    396448      // (2) to insert a new fact task entry for new tasks
     
    403455         join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
    404456         from lastFact in lastFactPerTask.DefaultIfEmpty()
    405          join client in dimClientDao.GetActiveClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
     457         join client in dimClientDao.GetAllOnlineClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
    406458         from client in clientsPerSlaveId.DefaultIfEmpty()
    407459         select new {
Note: See TracChangeset for help on using the changeset viewer.