Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/HiveStatistics/DimJobDao.cs @ 17574

Last change on this file since 17574 was 17574, checked in by jkarder, 4 years ago

#3062: overhauled statistics generation and cleanup

  • switched to a single thread for database cleanup and statistics generation (executed sequentially)
  • switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
  • added code that aborts tasks whose jobs have already been marked for deletion
  • added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
  • updated DAO methods and introduced more bare metal sql
  • introduced DAO methods for batch deletion
  • fixed usage of enum values in DAO sql queries
  • deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
  • changed scheduling for less interference with janitor and other heartbeats
    • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
    • changed signature of ITaskScheduler.Schedule
    • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
    • changed RoundRobinTaskScheduler to use bare metal sql
  • made MessageContainer a storable type (leftover)
  • updated HiveJanitorServiceInstaller.nsi
File size: 3.1 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Data.Linq;
24using System.Linq;
25
26namespace HeuristicLab.Services.Hive.DataAccess.Daos.HiveStatistics {
27  public class DimJobDao : GenericDao<Guid, DimJob> {
28    public DimJobDao(DataContext dataContext) : base(dataContext) { }
29
30    public override DimJob GetById(Guid id) {
31      return GetByIdQuery(DataContext, id);
32    }
33
34    public IQueryable<DimJob> GetByUserId(Guid id) {
35      return Table.Where(x => x.UserId == id);
36    }
37
38    public IQueryable<DimJob> GetNotCompletedJobs() {
39      return Table.Where(x => x.DateCompleted == null);
40    }
41
42    public IQueryable<DimJob> GetCompletedJobs() {
43      return Table.Where(x => x.DateCompleted != null);
44    }
45
46    public void UpdateExistingDimJobs() {
47      DataContext.ExecuteCommand(UpdateExistingDimJobsQuery);
48    }
49
50    #region Compiled queries
51    private static readonly Func<DataContext, Guid, DimJob> GetByIdQuery =
52      CompiledQuery.Compile((DataContext db, Guid id) =>
53        (from dimJob in db.GetTable<DimJob>()
54         where dimJob.JobId == id
55         select dimJob).SingleOrDefault());
56    #endregion
57
58    #region String queries
59    private const string UpdateExistingDimJobsQuery = @"
60UPDATE u
61SET
62  u.JobName = case when x.JobId is null then u.JobName else x.JobName end,
63  u.TotalTasks = x.TotalTasks,
64  u.CompletedTasks = x.CompletedTasks,
65  u.DateCompleted =
66    case when x.totaltasks = x.CompletedTasks
67      then (case when x.JobId is null and x.DateCompleted is null then GETDATE() else x.DateCompleted end)
68    else u.DateCompleted
69  end,
70  u.ProjectId = case when x.JobId is null then u.ProjectId else x.ProjectId end
71FROM [statistics].dimjob u
72JOIN (
73  SELECT
74    dj.JobId as DimJobId,
75    j.JobId as JobId,
76    j.Name as JobName,
77    COUNT(*) as TotalTasks,
78    SUM(
79    CASE
80    WHEN TaskState in ('Finished', 'Aborted', 'Failed') then 1
81    ELSE 0
82    END) as CompletedTasks,
83    MAX(EndTime) as DateCompleted,
84    dp.ProjectId as ProjectId
85  from [statistics].DimJob dj
86  join [statistics].FactTask ft on dj.JobId = ft.JobId
87  left join Job j on j.JobId = dj.JobId
88  left join [statistics].DimProject dp on j.ProjectId = dp.ProjectId
89  where dj.DateCompleted is null and dp.DateExpired is null
90  group by dj.JobId, j.JobId, j.Name, dp.ProjectId
91) as x on u.JobId = x.DimJobId";
92    #endregion
93  }
94}
Note: See TracBrowser for help on using the repository browser.