Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/TaskDao.cs

Last change on this file was 17574, checked in by jkarder, 4 years ago

#3062: overhauled statistics generation and cleanup

  • switched to a single thread for database cleanup and statistics generation (executed sequentially)
  • switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
  • added code that aborts tasks whose jobs have already been marked for deletion
  • added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
  • updated DAO methods and introduced more bare metal sql
  • introduced DAO methods for batch deletion
  • fixed usage of enum values in DAO sql queries
  • deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
  • changed scheduling for less interference with janitor and other heartbeats
    • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
    • changed signature of ITaskScheduler.Schedule
    • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
    • changed RoundRobinTaskScheduler to use bare metal sql
  • made MessageContainer a storable type (leftover)
  • updated HiveJanitorServiceInstaller.nsi
File size: 5.6 KB
RevLine 
[12468]1#region License Information
2/* HeuristicLab
[17180]3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[12468]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
[12691]23using System.Collections.Generic;
[12468]24using System.Data.Linq;
25using System.Linq;
26
27namespace HeuristicLab.Services.Hive.DataAccess.Daos {
28  public class TaskDao : GenericDao<Guid, Task> {
29    public TaskDao(DataContext dataContext) : base(dataContext) { }
30
31    public override Task GetById(Guid id) {
[12691]32      return GetByIdQuery(DataContext, id);
[12468]33    }
34
35    public IQueryable<Task> GetAllChildTasks() {
36      return Table.Where(x => !x.IsParentTask);
37    }
[12516]38
[12584]39    public IQueryable<Task> GetByJobId(Guid id) {
[12516]40      return Table.Where(x => x.JobId == id);
41    }
[12691]42    public class TaskPriorityInfo {
43      public Guid JobId { get; set; }
44      public Guid TaskId { get; set; }
45      public int Priority { get; set; }
46    }
47
48    public IEnumerable<TaskPriorityInfo> GetWaitingTasks(Slave slave) {
49      //Originally we checked here if there are parent tasks which should be calculated (with GetParentTasks(resourceIds, count, false);).
50      //Because there is at the moment no case where this makes sense (there don't exist parent tasks which need to be calculated),
51      //we skip this step because it's wasted runtime
[17574]52      return DataContext.ExecuteQuery<TaskPriorityInfo>(GetWaitingTasksQueryString,
53        slave.ResourceId,
54        Enum.GetName(typeof(TaskState), TaskState.Waiting),
55        slave.FreeCores,
[16117]56        slave.FreeMemory).ToList();
[12691]57    }
58
[12789]59    /// <summary>
60    /// returns all parent tasks which are waiting for their child tasks to finish
61    /// </summary>
62    /// <param name="resourceIds">list of resourceids which for which the task should be valid</param>
63    /// <param name="count">maximum number of task to return</param>
[16117]64    /// <param name="finished">if true, all parent tasks which have FinishWhenChildJobsFinished=true are returned, otherwise only FinishWhenChildJobsFinished=false are returned</param>
[12789]65    /// <returns></returns>
66    public IEnumerable<Task> GetParentTasks(IEnumerable<Guid> resourceIds, int count, bool finished) {
[17574]67      var query = from t in Table
68                  where t.State == TaskState.Waiting
69                      && t.IsParentTask
70                      && t.Job.AssignedJobResources.All(x => resourceIds.ToList().Contains(x.ResourceId))
71                      && t.FinishWhenChildJobsFinished == finished
72                      && t.ChildJobs.Any()
73                      && t.ChildJobs.All(x =>
74                        x.State == TaskState.Finished
75                        || x.State == TaskState.Aborted
76                        || x.State == TaskState.Failed)
[16117]77                  orderby t.Priority descending
78                  select t;
[12789]79      return count == 0 ? query.ToArray() : query.Take(count).ToArray();
80    }
81
[12691]82    public void UpdateExecutionTime(Guid taskId, double executionTime) {
83      DataContext.ExecuteCommand(UpdateExecutionTimeQuery, executionTime, DateTime.Now, taskId);
84    }
85
[17574]86    public int DeleteObsolete(int batchSize) {
87      return DataContext.ExecuteCommand(DeleteObsoleteQueryString, batchSize);
88    }
89
[12691]90    #region Compiled queries
91    private static readonly Func<DataContext, Guid, Task> GetByIdQuery =
92      CompiledQuery.Compile((DataContext db, Guid taskId) =>
93        (from task in db.GetTable<Task>()
94         where task.TaskId == taskId
95         select task).SingleOrDefault());
96    #endregion
97
98    #region String queries
[16117]99    private const string GetCalculatingChildTasksByProjectId = @"
100      SELECT t.* FROM [Task] t, [Job] j
101        WHERE t.IsParentTask = 0
102        AND t.TaskState = 'Calculating'
103        AND t.JobId = j.JobId
104        AND j.ProjectId = {0}
105        ORDER BY j.ProjectId
106    ";
[12691]107    private const string GetWaitingTasksQueryString = @"
[16117]108      WITH rbranch AS (
[12691]109        SELECT ResourceId, ParentResourceId
110        FROM [Resource]
111        WHERE ResourceId = {0}
112        UNION ALL
113        SELECT r.ResourceId, r.ParentResourceId
[16117]114        FROM [Resource] r
115        JOIN rbranch rb ON rb.ParentResourceId = r.ResourceId
[12691]116      )
117      SELECT DISTINCT t.TaskId, t.JobId, t.Priority
[16117]118      FROM [Task] t, [Job] j, [AssignedJobResource] ajr, rbranch
[12691]119      WHERE NOT (t.IsParentTask = 1 AND t.FinishWhenChildJobsFinished = 1)
[16117]120      AND t.TaskState = {1}
121      AND t.CoresNeeded <= {2}
122      AND t.MemoryNeeded <= {3}
123      AND t.JobId = j.JobId
124      AND j.JobState = 'Online'
125      AND j.JobId = ajr.JobId
126      AND ajr.ResourceId = rbranch.ResourceId
[12691]127    ";
128
129    private const string UpdateExecutionTimeQuery = @"
130      UPDATE [Task]
131         SET ExecutionTimeMs = {0},
132             LastHeartbeat = {1}
133       WHERE TaskId = {2}
134    ";
[17574]135
136    private const string DeleteObsoleteQueryString = @"
137delete top ({0}) t1
138from task t1
139  left join task t2 on t1.taskid = t2.parenttaskid
140  join job j on j.jobid = t1.jobid
141where j.jobstate = 'deletionpending' and t2.taskid is null
142";
[12691]143    #endregion
[12468]144  }
[12691]145}
Note: See TracBrowser for help on using the repository browser.