Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
05/29/20 13:28:25 (4 years ago)
Author:
jkarder
Message:

#3062: overhauled statistics generation and cleanup

  • switched to a single thread for database cleanup and statistics generation (executed sequentially)
  • switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
  • added code that aborts tasks whose jobs have already been marked for deletion
  • added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
  • updated DAO methods and introduced more bare metal sql
  • introduced DAO methods for batch deletion
  • fixed usage of enum values in DAO sql queries
  • deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
  • changed scheduling for less interference with janitor and other heartbeats
    • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
    • changed signature of ITaskScheduler.Schedule
    • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
    • changed RoundRobinTaskScheduler to use bare metal sql
  • made MessageContainer a storable type (leftover)
  • updated HiveJanitorServiceInstaller.nsi
File:
1 edited

Legend:

Unmodified
Added
Removed
  • trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/HiveStatistics/DimJobDao.cs

    r17180 r17574  
    4444    }
    4545
     46    public void UpdateExistingDimJobs() {
     47      DataContext.ExecuteCommand(UpdateExistingDimJobsQuery);
     48    }
     49
    4650    #region Compiled queries
    4751    private static readonly Func<DataContext, Guid, DimJob> GetByIdQuery =
     
    5155         select dimJob).SingleOrDefault());
    5256    #endregion
     57
     58    #region String queries
     59    private const string UpdateExistingDimJobsQuery = @"
     60UPDATE u
     61SET
     62  u.JobName = case when x.JobId is null then u.JobName else x.JobName end,
     63  u.TotalTasks = x.TotalTasks,
     64  u.CompletedTasks = x.CompletedTasks,
     65  u.DateCompleted =
     66    case when x.totaltasks = x.CompletedTasks
     67      then (case when x.JobId is null and x.DateCompleted is null then GETDATE() else x.DateCompleted end)
     68    else u.DateCompleted
     69  end,
     70  u.ProjectId = case when x.JobId is null then u.ProjectId else x.ProjectId end
     71FROM [statistics].dimjob u
     72JOIN (
     73  SELECT
     74    dj.JobId as DimJobId,
     75    j.JobId as JobId,
     76    j.Name as JobName,
     77    COUNT(*) as TotalTasks,
     78    SUM(
     79    CASE
     80    WHEN TaskState in ('Finished', 'Aborted', 'Failed') then 1
     81    ELSE 0
     82    END) as CompletedTasks,
     83    MAX(EndTime) as DateCompleted,
     84    dp.ProjectId as ProjectId
     85  from [statistics].DimJob dj
     86  join [statistics].FactTask ft on dj.JobId = ft.JobId
     87  left join Job j on j.JobId = dj.JobId
     88  left join [statistics].DimProject dp on j.ProjectId = dp.ProjectId
     89  where dj.DateCompleted is null and dp.DateExpired is null
     90  group by dj.JobId, j.JobId, j.Name, dp.ProjectId
     91) as x on u.JobId = x.DimJobId";
     92    #endregion
    5393  }
    5494}
Note: See TracChangeset for help on using the changeset viewer.