Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/JobDao.cs @ 18066

Last change on this file since 18066 was 17574, checked in by jkarder, 5 years ago

#3062: overhauled statistics generation and cleanup

  • switched to a single thread for database cleanup and statistics generation (executed sequentially)
  • switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
  • added code that aborts tasks whose jobs have already been marked for deletion
  • added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
  • updated DAO methods and introduced more bare metal sql
  • introduced DAO methods for batch deletion
  • fixed usage of enum values in DAO sql queries
  • deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
  • changed scheduling for less interference with janitor and other heartbeats
    • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
    • changed signature of ITaskScheduler.Schedule
    • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
    • changed RoundRobinTaskScheduler to use bare metal sql
  • made MessageContainer a storable type (leftover)
  • updated HiveJanitorServiceInstaller.nsi
File size: 4.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Data.Linq;
25using System.Linq;
26
27namespace HeuristicLab.Services.Hive.DataAccess.Daos {
28  public class JobDao : GenericDao<Guid, Job> {
29    public JobDao(DataContext dataContext) : base(dataContext) { }
30
31    public override Job GetById(Guid id) {
32      return GetByIdQuery(DataContext, id);
33    }
34
35    public void DeleteByState(JobState state) {
36      DataContext.ExecuteCommand(DeleteByStateQueryString, Enum.GetName(typeof(JobState), state));
37    }
38
39    public int DeleteByState(JobState state, int batchSize) {
40      return DataContext.ExecuteCommand(DeleteTopNByStateQueryString, batchSize, Enum.GetName(typeof(JobState), state));
41    }
42
43    public IEnumerable<Job> GetByProjectId(Guid id) {
44      return GetByProjectIdQuery(DataContext, id);
45    }
46
47    public IEnumerable<Job> GetByProjectIds(IEnumerable<Guid> projectIds) {
48      string paramProjectIds = string.Join(",", projectIds.ToList().Select(x => string.Format("'{0}'", x)));
49      if (!string.IsNullOrWhiteSpace(paramProjectIds)) {
50        string queryString = string.Format(GetByProjectIdsQueryString, paramProjectIds);
51        return DataContext.ExecuteQuery<Job>(queryString);
52      }
53      return Enumerable.Empty<Job>();
54    }
55
56    public IEnumerable<Job> GetByState(JobState state) {
57      return GetByStateQuery(DataContext, state.ToString());
58    }
59
60    public IEnumerable<Guid> GetJobIdsByState(JobState state) {
61      return GetJobIdsByStateQuery(DataContext, state.ToString());
62    }
63
64    public IEnumerable<Job> GetJobsReadyForDeletion() {
65      return GetJobsReadyForDeletionQuery(DataContext);
66    }
67
68
69    #region Compiled queries
70    private static readonly Func<DataContext, Guid, Job> GetByIdQuery =
71      CompiledQuery.Compile((DataContext db, Guid jobId) =>
72        (from job in db.GetTable<Job>()
73         where job.JobId == jobId
74         select job).SingleOrDefault());
75    private static readonly Func<DataContext, Guid, IEnumerable<Job>> GetByProjectIdQuery =
76      CompiledQuery.Compile((DataContext db, Guid projectId) =>
77        (from job in db.GetTable<Job>()
78         where job.ProjectId == projectId
79         select job));
80    private static readonly Func<DataContext, string, IEnumerable<Job>> GetByStateQuery =
81      CompiledQuery.Compile((DataContext db, string jobState) =>
82        (from job in db.GetTable<Job>()
83         where job.State.ToString() == jobState
84         select job));
85    private static readonly Func<DataContext, string, IEnumerable<Guid>> GetJobIdsByStateQuery =
86      CompiledQuery.Compile((DataContext db, string jobState) =>
87        (from job in db.GetTable<Job>()
88         where job.State.ToString() == jobState
89         select job.JobId));
90    private static readonly Func<DataContext, IEnumerable<Job>> GetJobsReadyForDeletionQuery =
91      CompiledQuery.Compile((DataContext db) =>
92        (from job in db.GetTable<Job>()
93         where job.State == JobState.StatisticsPending
94               && (from task in db.GetTable<Task>()
95                   where task.JobId == job.JobId
96                   select task).All(x => x.State == TaskState.Finished
97                                      || x.State == TaskState.Aborted
98                                      || x.State == TaskState.Failed)
99         select job));
100    #endregion
101
102    #region String queries
103    private const string DeleteByStateQueryString = @"
104      DELETE FROM [Job]
105      WHERE JobState = {0}
106    ";
107    private const string DeleteTopNByStateQueryString = @"
108      DELETE TOP ({0})
109      FROM [Job]
110      WHERE JobState = {1}
111    ";
112    private const string GetStatisticsPendingJobs = @"
113      SELECT DISTINCT j.*
114      FROM [Job] j
115      WHERE j.JobState = 'StatisticsPending'
116      AND 'Calculating' NOT IN (
117        SELECT t.TaskState
118        FROM [Task] t
119        WHERE t.JobId = j.JobId)
120    ";
121    private const string GetByProjectIdsQueryString = @"
122      SELECT DISTINCT j.*
123      FROM [Job] j
124      WHERE j.ProjectId IN ({0})
125    ";
126    #endregion
127  }
128}
Note: See TracBrowser for help on using the repository browser.