Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HivePerformance/sources/HeuristicLab.Services.Hive/3.3/OptimizedHiveDao.cs @ 9485

Last change on this file since 9485 was 9485, checked in by pfleck, 10 years ago

#2030

Fixed Bugs in Shutdown Queries.

Added SQL script for migrating to filestream based storage of task and plugin data.

To remove files after data removal turned on Auto-Shrink and set recovery-model to Simple.
Also set a checkpoint in the database.

File size: 9.9 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2012 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
24using System.Data.Linq;
25using System.Linq;
26using DT = HeuristicLab.Services.Hive.DataTransfer;
27
28namespace HeuristicLab.Services.Hive.DataAccess {
29  public class OptimizedHiveDao : IOptimizedHiveDao {
30    private static HiveDataContext Db { get { return HiveOperationContext.Current.DataContext; } }
31
32    #region Task Methods
33    public Task GetTaskById(Guid taskId) {
34      return GetTaskByIdQuery(Db, taskId).SingleOrDefault();
35    }
36
37    private static readonly Func<HiveDataContext, Guid, IQueryable<Task>> GetTaskByIdQuery = CompiledQuery.Compile((HiveDataContext db, Guid taskId) =>
38      from t in db.Tasks
39      where t.TaskId == taskId
40      select t
41    );
42
43    public Task GetTaskByDto(DT.Task taskDto) {
44      var task = GetTaskById(taskDto.Id);
45      DT.Convert.ToEntity(taskDto, task);
46      return task;
47    }
48
49    public Tuple<Task, Guid?> GetTaskByIdAndLastStateLogSlaveId(Guid taskId) {
50      return GetTaskByIdAndLastStateLogSlaveIdQuery(Db, taskId).SingleOrDefault();
51    }
52
53    private static readonly Func<HiveDataContext, Guid, IQueryable<Tuple<Task, Guid?>>> GetTaskByIdAndLastStateLogSlaveIdQuery = CompiledQuery.Compile((HiveDataContext db, Guid taskId) =>
54       from t in db.Tasks
55       join sl in db.StateLogs on t.TaskId equals sl.TaskId
56       where t.TaskId == taskId
57       orderby sl.DateTime descending
58       group sl by t into logs
59       select new Tuple<Task, Guid?>(logs.Key, logs.First().SlaveId)
60    );
61
62    private const string GetWaitingTasksQueryString = @"
63      WITH pr AS (
64        SELECT ResourceId, ParentResourceId
65        FROM [Resource]
66        WHERE ResourceId = {0}
67        UNION ALL
68        SELECT r.ResourceId, r.ParentResourceId
69        FROM [Resource] r JOIN pr ON r.ResourceId = pr.ParentResourceId
70      )
71      SELECT DISTINCT t.TaskId, t.JobId, t.Priority
72      FROM pr JOIN AssignedResources ar ON ar.ResourceId = pr.ResourceId
73          JOIN Task t ON t.TaskId = ar.TaskId
74      WHERE NOT (t.IsParentTask = 1 AND t.FinishWhenChildJobsFinished = 1)
75          AND t.TaskState = {1}
76          AND t.CoresNeeded <= {2}
77          AND t.MemoryNeeded <= {3}
78    ";
79
80    public IEnumerable<TaskInfoForScheduler> GetWaitingTasks(Slave slave) {
81      //Originally we checked here if there are parent tasks which should be calculated (with GetParentTasks(resourceIds, count, false);).
82      //Because there is at the moment no case where this makes sense (there don't exist parent tasks which need to be calculated),
83      //we skip this step because it's wasted runtime
84      return Db.ExecuteQuery<TaskInfoForScheduler>(GetWaitingTasksQueryString, slave.ResourceId, Enum.GetName(typeof(TaskState), TaskState.Waiting), slave.FreeCores, slave.FreeMemory);
85    }
86
87    public IQueryable<DT.LightweightTask> GetLightweightTasks(Guid jobId) {
88      return GetLightweightTasksQuery(Db, jobId);
89    }
90
91    private static readonly Func<HiveDataContext, Guid, IQueryable<DT.LightweightTask>> GetLightweightTasksQuery = CompiledQuery.Compile((HiveDataContext db, Guid jobId) =>
92        from task in db.Tasks
93        where task.JobId == jobId
94        select new DT.LightweightTask {
95          Id = task.TaskId,
96          ExecutionTime = TimeSpan.FromMilliseconds(task.ExecutionTimeMs),
97          ParentTaskId = task.ParentTaskId,
98          StateLog = task.StateLogs.Select(sl => ConvertStateLog(sl)).ToList(),
99          State = ConvertTaskState(task.State),
100          Command = ConvertCommand(task.Command),
101          LastTaskDataUpdate = task.JobData.LastUpdate
102        }
103    );
104
105    private static readonly Func<StateLog, DT.StateLog> ConvertStateLog = sl => DT.Convert.ToDto(sl);
106    private static readonly Func<TaskState, DT.TaskState> ConvertTaskState = ts => DT.Convert.ToDto(ts);
107    private static readonly Func<Command?, DT.Command?> ConvertCommand = c => DT.Convert.ToDto(c);
108
109    public void UpdateTask(Task task) {
110      Db.SubmitChanges();
111    }
112
113    public Task UpdateTaskState(Guid taskId, TaskState taskState, Guid? slaveId, Guid? userId, string exception) {
114      Db.StateLogs.InsertOnSubmit(new StateLog {
115        TaskId = taskId,
116        State = taskState,
117        SlaveId = slaveId,
118        UserId = userId,
119        Exception = exception,
120        DateTime = DateTime.Now
121      });
122
123      var task = GetTaskById(taskId);
124      task.State = taskState;
125
126      Db.SubmitChanges();
127
128      return task;
129    }
130
131    public Guid AddTask(Task task) {
132      Db.Tasks.InsertOnSubmit(task);
133      Db.SubmitChanges();
134      return task.TaskId;
135    }
136
137    public void AssignJobToResource(Guid taskId, IEnumerable<Guid> resourceIds) {
138      Db.AssignedResources.InsertAllOnSubmit(resourceIds.Select(resourceId => new AssignedResource { TaskId = taskId, ResourceId = resourceId }));
139      Db.SubmitChanges();
140    }
141
142    private const string TaskIsAllowedToBeCalculatedBySlaveQueryString = @"
143      WITH pr AS (
144        SELECT ResourceId, ParentResourceId
145        FROM [Resource]
146        WHERE ResourceId = {0}
147        UNION ALL
148        SELECT r.ResourceId, r.ParentResourceId
149        FROM [Resource] r JOIN pr ON r.ResourceId = pr.ParentResourceId
150      )
151      SELECT COUNT(ar.TaskId)
152      FROM pr JOIN AssignedResources ar ON pr.ResourceId = ar.ResourceId
153      WHERE ar.TaskId = {1}
154    ";
155
156    public bool TaskIsAllowedToBeCalculatedBySlave(Guid taskId, Guid slaveId) {
157      return Db.ExecuteQuery<int>(TaskIsAllowedToBeCalculatedBySlaveQueryString, slaveId, taskId).First() > 0;
158    }
159    #endregion
160
161    #region TaskData Methods
162    public TaskData GetTaskDataById(Guid id) {
163      return GetTaskDataByIdQuery(Db, id).SingleOrDefault();
164    }
165
166    private static readonly Func<HiveDataContext, Guid, IQueryable<TaskData>> GetTaskDataByIdQuery = CompiledQuery.Compile((HiveDataContext db, Guid id) =>
167      from t in db.TaskDatas
168      where t.TaskId == id
169      select t
170    );
171
172    public TaskData GetTaskDataByDto(DT.TaskData dto) {
173      var taskData = GetTaskDataById(dto.TaskId);
174      DT.Convert.ToEntity(dto, taskData);
175      return taskData;
176    }
177
178    public void UpdateTaskData(TaskData taskData) {
179      Db.SubmitChanges();
180    }
181    #endregion
182
183    #region StateLog Methods
184
185    #endregion
186
187    #region Job Methods
188
189    #endregion
190
191    #region JobPermission Methods
192
193    #endregion
194
195    #region Plugin Methods
196    public Plugin GetPluginById(Guid pluginId) {
197      return GetPluginByIdQuery(Db, pluginId).SingleOrDefault();
198    }
199
200    private static readonly Func<HiveDataContext, Guid, IQueryable<Plugin>> GetPluginByIdQuery = CompiledQuery.Compile((HiveDataContext db, Guid pluginId) =>
201      from p in db.Plugins
202      where p.PluginId == pluginId
203      select p
204    );
205
206    #endregion
207
208    #region PluginData Methods
209
210    #endregion
211
212    #region Slave Methods
213    public Slave GetSlaveById(Guid id) {
214      return GetSlaveByIdQuery(Db, id).SingleOrDefault();
215    }
216
217    private static readonly Func<HiveDataContext, Guid, IQueryable<Slave>> GetSlaveByIdQuery = CompiledQuery.Compile((HiveDataContext db, Guid slaveId) =>
218      from s in db.Resources.OfType<Slave>()
219      where s.ResourceId == slaveId
220      select s
221    );
222
223    public void UpdateSlave(Slave slave) {
224      Db.SubmitChanges();
225    }
226
227    private const string DowntimeQueryString = @"
228      WITH pr AS (
229        SELECT ResourceId, ParentResourceId
230        FROM [Resource]
231        WHERE ResourceId = {0}
232        UNION ALL
233        SELECT r.ResourceId, r.ParentResourceId
234        FROM [Resource] r JOIN pr ON r.ResourceId = pr.ParentResourceId
235      )
236      SELECT COUNT(dt.DowntimeId)
237      FROM pr JOIN [Downtime] dt ON pr.ResourceId = dt.ResourceId
238      WHERE {1} BETWEEN dt.StartDate AND dt.EndDate
239        AND dt.DowntimeType = {2}
240      ";
241
242    public bool SlaveHasToShutdownComputer(Guid slaveId) {
243      return Db.ExecuteQuery<int>(DowntimeQueryString, slaveId, DateTime.Now, DowntimeType.Shutdown.ToString()).FirstOrDefault() > 0;
244    }
245
246    public bool SlaveIsAllowedToCalculate(Guid slaveId) {
247      return Db.ExecuteQuery<int>(DowntimeQueryString, slaveId, DateTime.Now, DowntimeType.Offline.ToString()).FirstOrDefault() == 0;
248    }
249    #endregion
250
251    #region SlaveGroup Methods
252
253    #endregion
254
255    #region Resource Methods
256    public IEnumerable<Guid> GetAssignedResourceIds(Guid taskId) {
257      return GetAssignedResourceIdsQuery(Db, taskId);
258    }
259
260    private static readonly Func<HiveDataContext, Guid, IQueryable<Guid>> GetAssignedResourceIdsQuery = CompiledQuery.Compile((HiveDataContext db, Guid taskId) =>
261      from ar in db.AssignedResources
262      where ar.TaskId == taskId
263      select ar.ResourceId
264    );
265    #endregion
266
267    #region ResourcePermission Methods
268
269    #endregion
270
271    #region Authorization Methods
272
273    #endregion
274
275    #region Lifecycle Methods
276
277    #endregion
278
279    #region Downtime Methods
280    #endregion
281
282    #region Statistics Methods
283
284    #endregion
285
286    #region UserPriority Methods
287
288    #endregion
289
290    #region Helpers
291
292    #endregion
293  }
294}
Note: See TracBrowser for help on using the repository browser.