Free cookie consent management tool by TermsFeed Policy Generator

source: trunk/HeuristicLab.Services.Hive.DataAccess/3.3/Manager/PersistenceManager.cs @ 18066

Last change on this file since 18066 was 17574, checked in by jkarder, 5 years ago

#3062: overhauled statistics generation and cleanup

  • switched to a single thread for database cleanup and statistics generation (executed sequentially)
  • switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
  • added code that aborts tasks whose jobs have already been marked for deletion
  • added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
  • updated DAO methods and introduced more bare metal sql
  • introduced DAO methods for batch deletion
  • fixed usage of enum values in DAO sql queries
  • deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
  • changed scheduling for less interference with janitor and other heartbeats
    • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
    • changed signature of ITaskScheduler.Schedule
    • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
    • changed RoundRobinTaskScheduler to use bare metal sql
  • made MessageContainer a storable type (leftover)
  • updated HiveJanitorServiceInstaller.nsi
File size: 9.8 KB
RevLine 
[12468]1#region License Information
2/* HeuristicLab
[17180]3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
[12468]4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Data.Linq;
[12761]24using System.Linq;
[12468]25using System.Transactions;
26using HeuristicLab.Services.Hive.DataAccess.Daos;
27using HeuristicLab.Services.Hive.DataAccess.Daos.HiveStatistics;
[12761]28using HeuristicLab.Services.Hive.DataAccess.Data;
[12468]29using HeuristicLab.Services.Hive.DataAccess.Interfaces;
30
31namespace HeuristicLab.Services.Hive.DataAccess.Manager {
32  public class PersistenceManager : IPersistenceManager {
33    private readonly DataContext dataContext;
34    public DataContext DataContext {
35      get { return dataContext; }
36    }
37
38    #region Hive daos
[12691]39
[16117]40    private AssignedJobResourceDao assignedJobResourceDao;
41    public AssignedJobResourceDao AssignedJobResourceDao {
42      get { return assignedJobResourceDao ?? (assignedJobResourceDao = new AssignedJobResourceDao(dataContext)); }
[12468]43    }
44
[16117]45    private AssignedProjectResourceDao assignedProjectResourceDao;
46    public AssignedProjectResourceDao AssignedProjectResourceDao {
47      get { return assignedProjectResourceDao ?? (assignedProjectResourceDao = new AssignedProjectResourceDao(dataContext)); }
48    }
49
[12691]50    private DowntimeDao downtimeDao;
[12468]51    public DowntimeDao DowntimeDao {
[12691]52      get { return downtimeDao ?? (downtimeDao = new DowntimeDao(dataContext)); }
[12468]53    }
54
[12691]55    private JobDao jobDao;
[12468]56    public JobDao JobDao {
[12691]57      get { return jobDao ?? (jobDao = new JobDao(dataContext)); }
[12468]58    }
59
[12691]60    private JobPermissionDao jobPermissionDao;
[12468]61    public JobPermissionDao JobPermissionDao {
[12691]62      get { return jobPermissionDao ?? (jobPermissionDao = new JobPermissionDao(dataContext)); }
[12468]63    }
64
[12691]65    private LifecycleDao lifecycleDao;
[12468]66    public LifecycleDao LifecycleDao {
[12691]67      get { return lifecycleDao ?? (lifecycleDao = new LifecycleDao(dataContext)); }
[12468]68    }
69
[12691]70    private PluginDao pluginDao;
[12468]71    public PluginDao PluginDao {
[12691]72      get { return pluginDao ?? (pluginDao = new PluginDao(dataContext)); }
[12468]73    }
74
[12691]75    private PluginDataDao pluginDataDao;
[12468]76    public PluginDataDao PluginDataDao {
[12691]77      get { return pluginDataDao ?? (pluginDataDao = new PluginDataDao(dataContext)); }
[12468]78    }
79
[16117]80    private ProjectDao projectDao;
81    public ProjectDao ProjectDao {
82      get { return projectDao ?? (projectDao = new ProjectDao(dataContext)); }
83    }
84
85    private ProjectPermissionDao projectPermissionDao;
86    public ProjectPermissionDao ProjectPermissionDao {
87      get { return projectPermissionDao ?? (projectPermissionDao = new ProjectPermissionDao(dataContext)); }
88    }
89
[12691]90    private RequiredPluginDao requiredPluginDao;
[12468]91    public RequiredPluginDao RequiredPluginDao {
[12691]92      get { return requiredPluginDao ?? (requiredPluginDao = new RequiredPluginDao(dataContext)); }
[12468]93    }
94
[12691]95    private ResourceDao resourceDao;
[12468]96    public ResourceDao ResourceDao {
[12691]97      get { return resourceDao ?? (resourceDao = new ResourceDao(dataContext)); }
[12468]98    }
99
[12691]100    private SlaveDao slaveDao;
[12468]101    public SlaveDao SlaveDao {
[12691]102      get { return slaveDao ?? (slaveDao = new SlaveDao(dataContext)); }
[12468]103    }
104
[12691]105    private SlaveGroupDao slaveGroupDao;
[12468]106    public SlaveGroupDao SlaveGroupDao {
[12691]107      get { return slaveGroupDao ?? (slaveGroupDao = new SlaveGroupDao(dataContext)); }
[12468]108    }
109
[12691]110    private StateLogDao stateLogDao;
[12468]111    public StateLogDao StateLogDao {
[12691]112      get { return stateLogDao ?? (stateLogDao = new StateLogDao(dataContext)); }
[12468]113    }
114
[12691]115    private TaskDao taskDao;
[12468]116    public TaskDao TaskDao {
[12691]117      get { return taskDao ?? (taskDao = new TaskDao(dataContext)); }
[12468]118    }
119
[12691]120    private TaskDataDao taskDataDao;
[12468]121    public TaskDataDao TaskDataDao {
[12691]122      get { return taskDataDao ?? (taskDataDao = new TaskDataDao(dataContext)); }
[12468]123    }
124
[12691]125    private UserPriorityDao userPriorityDao;
[12468]126    public UserPriorityDao UserPriorityDao {
[12691]127      get { return userPriorityDao ?? (userPriorityDao = new UserPriorityDao(dataContext)); }
[12468]128    }
129    #endregion
130
131    #region HiveStatistics daos
[12691]132
133    private DimClientDao dimClientDao;
[12468]134    public DimClientDao DimClientDao {
[12691]135      get { return dimClientDao ?? (dimClientDao = new DimClientDao(dataContext)); }
[12468]136    }
137
[12691]138    private DimJobDao dimJobDao;
[12468]139    public DimJobDao DimJobDao {
[12691]140      get { return dimJobDao ?? (dimJobDao = new DimJobDao(dataContext)); }
[12468]141    }
142
[12691]143    private DimTimeDao dimTimeDao;
[12468]144    public DimTimeDao DimTimeDao {
[12691]145      get { return dimTimeDao ?? (dimTimeDao = new DimTimeDao(dataContext)); }
[12468]146    }
147
[12691]148    private DimUserDao dimUserDao;
[12468]149    public DimUserDao DimUserDao {
[12691]150      get { return dimUserDao ?? (dimUserDao = new DimUserDao(dataContext)); }
[12468]151    }
152
[12691]153    private FactClientInfoDao factClientInfoDao;
[12468]154    public FactClientInfoDao FactClientInfoDao {
[12691]155      get { return factClientInfoDao ?? (factClientInfoDao = new FactClientInfoDao(dataContext)); }
[12468]156    }
157
[12691]158    private FactTaskDao factTaskDao;
[12468]159    public FactTaskDao FactTaskDao {
[12691]160      get { return factTaskDao ?? (factTaskDao = new FactTaskDao(dataContext)); }
[12468]161    }
[16117]162
163    private DimProjectDao dimProjectDao;
164    public DimProjectDao DimProjectDao {
165      get { return dimProjectDao ?? (dimProjectDao = new DimProjectDao(dataContext)); }
166    }
167
168    private FactProjectInfoDao factProjectInfoDao;
169    public FactProjectInfoDao FactProjectInfoDao {
170      get { return factProjectInfoDao ?? (factProjectInfoDao = new FactProjectInfoDao(dataContext)); }
171    }
[12468]172    #endregion
173
174    public PersistenceManager(bool longRunning = false) {
175      var context = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString);
176      if (longRunning) context.CommandTimeout = (int)Settings.Default.LongRunningDatabaseCommandTimeout.TotalSeconds;
177      dataContext = context;
178    }
179
[12691]180    public PersistenceManager(DataContext dataContext) {
181      this.dataContext = dataContext;
182    }
183
[12468]184    #region Transaction management
185    public void UseTransaction(Action call, bool repeatableRead = false, bool longRunning = false) {
[17574]186      UseTransaction<object>(() => { call(); return null; }, repeatableRead, longRunning);
[12468]187    }
188
[17574]189    public void UseTransactionAndSubmit(Action call, bool repeatableRead = false, bool longRunning = false) {
190      UseTransaction(() => { call(); SubmitChanges(); }, repeatableRead, longRunning);
191    }
192
[12468]193    public T UseTransaction<T>(Func<T> call, bool repeatableRead = false, bool longRunning = false) {
194      int n = 10;
195      while (n > 0) {
196        TransactionScope transaction = CreateTransaction(repeatableRead, longRunning);
197        try {
198          T result = call();
199          transaction.Complete();
200          return result;
201        }
202        catch (System.Data.SqlClient.SqlException e) {
203          n--; // probably deadlock situation, let it roll back and repeat the transaction n times
204          LogFactory.GetLogger(typeof(TransactionManager).Namespace).Log(string.Format("Exception occured, repeating transaction {0} more times. Details: {1}", n, e.ToString()));
205          if (n <= 0) throw;
206        }
207        finally {
208          transaction.Dispose();
209        }
210      }
211      throw new Exception("Transaction couldn't be completed.");
212    }
213
[17574]214    public T UseTransactionAndSubmit<T>(Func<T> call, bool repeatableRead = false, bool longRunning = false) {
215      return UseTransaction(() => { var res = call(); SubmitChanges(); return res; }, repeatableRead, longRunning);
216    }
217
[12468]218    private static TransactionScope CreateTransaction(bool repeatableRead, bool longRunning) {
219      var options = new TransactionOptions {
220        IsolationLevel = repeatableRead ? IsolationLevel.RepeatableRead : IsolationLevel.ReadUncommitted
221      };
222      if (longRunning) {
223        options.Timeout = Settings.Default.LongRunningDatabaseCommandTimeout;
224      }
225      return new TransactionScope(TransactionScopeOption.Required, options);
226    }
227    #endregion
228
[12761]229    public TableInformation GetTableInformation(string table) {
[12883]230      string query = string.Format("sp_spaceused '{0}'", table);
[12761]231      var result = dataContext.ExecuteQuery<SqlServerTableInformation>(query).FirstOrDefault();
232      if (result == null) return null;
233      return new TableInformation {
234        Name = result.Name,
235        Rows = int.Parse(result.Rows.Remove(result.Rows.IndexOf(' '))),
236        Reserved = int.Parse(result.Reserved.Remove(result.Reserved.IndexOf(' '))),
237        Data = int.Parse(result.Data.Remove(result.Data.IndexOf(' '))),
238        IndexSize = int.Parse(result.Index_Size.Remove(result.Index_Size.IndexOf(' '))),
239        Unused = int.Parse(result.Unused.Remove(result.Unused.IndexOf(' ')))
240      };
241    }
242
[12468]243    public void SubmitChanges() {
244      if (dataContext != null) {
245        dataContext.SubmitChanges();
246      }
247    }
248
249    public void Dispose() {
[12853]250      if (dataContext != null) {
251        dataContext.Dispose();
252      }
[12468]253    }
[12761]254
255    private class SqlServerTableInformation {
256      public string Name { get; set; }
257      public string Rows { get; set; }
258      public string Reserved { get; set; }
259      public string Data { get; set; }
260      public string Index_Size { get; set; } // naming of sp_spaceused...
261      public string Unused { get; set; }
262    }
[12468]263  }
264}
Note: See TracBrowser for help on using the repository browser.