Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/TaskDao.cs @ 18095

Last change on this file since 18095 was 17574, checked in by jkarder, 5 years ago

#3062: overhauled statistics generation and cleanup

  • switched to a single thread for database cleanup and statistics generation (executed sequentially)
  • switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
  • added code that aborts tasks whose jobs have already been marked for deletion
  • added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
  • updated DAO methods and introduced more bare metal sql
  • introduced DAO methods for batch deletion
  • fixed usage of enum values in DAO sql queries
  • deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
  • changed scheduling for less interference with janitor and other heartbeats
    • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
    • changed signature of ITaskScheduler.Schedule
    • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
    • changed RoundRobinTaskScheduler to use bare metal sql
  • made MessageContainer a storable type (leftover)
  • updated HiveJanitorServiceInstaller.nsi
File size: 5.6 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Data.Linq;
25using System.Linq;
26
27namespace HeuristicLab.Services.Hive.DataAccess.Daos {
28  public class TaskDao : GenericDao<Guid, Task> {
29    public TaskDao(DataContext dataContext) : base(dataContext) { }
30
31    public override Task GetById(Guid id) {
32      return GetByIdQuery(DataContext, id);
33    }
34
35    public IQueryable<Task> GetAllChildTasks() {
36      return Table.Where(x => !x.IsParentTask);
37    }
38
39    public IQueryable<Task> GetByJobId(Guid id) {
40      return Table.Where(x => x.JobId == id);
41    }
42    public class TaskPriorityInfo {
43      public Guid JobId { get; set; }
44      public Guid TaskId { get; set; }
45      public int Priority { get; set; }
46    }
47
48    public IEnumerable<TaskPriorityInfo> GetWaitingTasks(Slave slave) {
49      //Originally we checked here if there are parent tasks which should be calculated (with GetParentTasks(resourceIds, count, false);).
50      //Because there is at the moment no case where this makes sense (there don't exist parent tasks which need to be calculated),
51      //we skip this step because it's wasted runtime
52      return DataContext.ExecuteQuery<TaskPriorityInfo>(GetWaitingTasksQueryString,
53        slave.ResourceId,
54        Enum.GetName(typeof(TaskState), TaskState.Waiting),
55        slave.FreeCores,
56        slave.FreeMemory).ToList();
57    }
58
59    /// <summary>
60    /// returns all parent tasks which are waiting for their child tasks to finish
61    /// </summary>
62    /// <param name="resourceIds">list of resourceids which for which the task should be valid</param>
63    /// <param name="count">maximum number of task to return</param>
64    /// <param name="finished">if true, all parent tasks which have FinishWhenChildJobsFinished=true are returned, otherwise only FinishWhenChildJobsFinished=false are returned</param>
65    /// <returns></returns>
66    public IEnumerable<Task> GetParentTasks(IEnumerable<Guid> resourceIds, int count, bool finished) {
67      var query = from t in Table
68                  where t.State == TaskState.Waiting
69                      && t.IsParentTask
70                      && t.Job.AssignedJobResources.All(x => resourceIds.ToList().Contains(x.ResourceId))
71                      && t.FinishWhenChildJobsFinished == finished
72                      && t.ChildJobs.Any()
73                      && t.ChildJobs.All(x =>
74                        x.State == TaskState.Finished
75                        || x.State == TaskState.Aborted
76                        || x.State == TaskState.Failed)
77                  orderby t.Priority descending
78                  select t;
79      return count == 0 ? query.ToArray() : query.Take(count).ToArray();
80    }
81
82    public void UpdateExecutionTime(Guid taskId, double executionTime) {
83      DataContext.ExecuteCommand(UpdateExecutionTimeQuery, executionTime, DateTime.Now, taskId);
84    }
85
86    public int DeleteObsolete(int batchSize) {
87      return DataContext.ExecuteCommand(DeleteObsoleteQueryString, batchSize);
88    }
89
90    #region Compiled queries
91    private static readonly Func<DataContext, Guid, Task> GetByIdQuery =
92      CompiledQuery.Compile((DataContext db, Guid taskId) =>
93        (from task in db.GetTable<Task>()
94         where task.TaskId == taskId
95         select task).SingleOrDefault());
96    #endregion
97
98    #region String queries
99    private const string GetCalculatingChildTasksByProjectId = @"
100      SELECT t.* FROM [Task] t, [Job] j
101        WHERE t.IsParentTask = 0
102        AND t.TaskState = 'Calculating'
103        AND t.JobId = j.JobId
104        AND j.ProjectId = {0}
105        ORDER BY j.ProjectId
106    ";
107    private const string GetWaitingTasksQueryString = @"
108      WITH rbranch AS (
109        SELECT ResourceId, ParentResourceId
110        FROM [Resource]
111        WHERE ResourceId = {0}
112        UNION ALL
113        SELECT r.ResourceId, r.ParentResourceId
114        FROM [Resource] r
115        JOIN rbranch rb ON rb.ParentResourceId = r.ResourceId
116      )
117      SELECT DISTINCT t.TaskId, t.JobId, t.Priority
118      FROM [Task] t, [Job] j, [AssignedJobResource] ajr, rbranch
119      WHERE NOT (t.IsParentTask = 1 AND t.FinishWhenChildJobsFinished = 1)
120      AND t.TaskState = {1}
121      AND t.CoresNeeded <= {2}
122      AND t.MemoryNeeded <= {3}
123      AND t.JobId = j.JobId
124      AND j.JobState = 'Online'
125      AND j.JobId = ajr.JobId
126      AND ajr.ResourceId = rbranch.ResourceId
127    ";
128
129    private const string UpdateExecutionTimeQuery = @"
130      UPDATE [Task]
131         SET ExecutionTimeMs = {0},
132             LastHeartbeat = {1}
133       WHERE TaskId = {2}
134    ";
135
136    private const string DeleteObsoleteQueryString = @"
137delete top ({0}) t1
138from task t1
139  left join task t2 on t1.taskid = t2.parenttaskid
140  join job j on j.jobid = t1.jobid
141where j.jobstate = 'deletionpending' and t2.taskid is null
142";
143    #endregion
144  }
145}
Note: See TracBrowser for help on using the repository browser.