Free cookie consent management tool by TermsFeed Policy Generator

source: branches/3082_Scaling/HeuristicLab.Services.Hive/3.3/HiveJanitor.cs

Last change on this file was 17574, checked in by jkarder, 4 years ago

#3062: overhauled statistics generation and cleanup

  • switched to a single thread for database cleanup and statistics generation (executed sequentially)
  • switched to preemptive deletion of items that are in status DeletionPending (for jobs: statelogs, taskdata, tasks)
  • added code that aborts tasks whose jobs have already been marked for deletion
  • added method UseTransactionAndSubmit in addition to UseTransaction in PersistenceManager
  • updated DAO methods and introduced more bare metal sql
  • introduced DAO methods for batch deletion
  • fixed usage of enum values in DAO sql queries
  • deleted unnecessary triggers tr_JobDeleteCascade and tr_TaskDeleteCascade in Prepare Hive Database.sql
  • changed scheduling for less interference with janitor and other heartbeats
    • increased scheduling patience from 20 to 70 seconds (to wait longer to get the mutex for scheduling)
    • changed signature of ITaskScheduler.Schedule
    • added base class for TaskSchedulers and moved assignment of tasks to slaves into it
    • changed RoundRobinTaskScheduler to use bare metal sql
  • made MessageContainer a storable type (leftover)
  • updated HiveJanitorServiceInstaller.nsi
File size: 3.4 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Threading;
24using HeuristicLab.Services.Hive.DataAccess;
25using HeuristicLab.Services.Hive.DataAccess.Interfaces;
26
27namespace HeuristicLab.Services.Hive {
28  public class HiveJanitor {
29    private bool stop;
30    private AutoResetEvent runWaitHandle;
31
32    private IPersistenceManager PersistenceManager {
33      get { return ServiceLocator.Instance.PersistenceManager; }
34    }
35    private IEventManager EventManager {
36      get { return ServiceLocator.Instance.EventManager; }
37    }
38
39    private IStatisticsGenerator StatisticsGenerator {
40      get { return ServiceLocator.Instance.StatisticsGenerator; }
41    }
42
43    public HiveJanitor() {
44      stop = false;
45      runWaitHandle = new AutoResetEvent(false);
46    }
47
48    public void StopJanitor() {
49      stop = true;
50      runWaitHandle.Set();
51    }
52
53    public void Run() {
54      while (!stop) {
55        RunCleanup();
56        RunGenerateStatistics();
57        runWaitHandle.WaitOne(Properties.Settings.Default.GenerateStatisticsInterval);
58      }
59      runWaitHandle.Close();
60    }
61
62    public void RunCleanup() {
63      var pm = PersistenceManager;
64      try {
65        LogFactory.GetLogger(typeof(HiveJanitor).Namespace).Log("HiveJanitor: starting cleanup.");
66        bool cleanup = false;
67
68        var lifecycleDao = pm.LifecycleDao;
69        pm.UseTransaction(() => {
70          var lifecycle = lifecycleDao.GetLastLifecycle();
71          if (lifecycle == null
72              || DateTime.Now - lifecycle.LastCleanup > Properties.Settings.Default.CleanupInterval) {
73            lifecycleDao.UpdateLifecycle();
74            cleanup = true;
75          }
76          pm.SubmitChanges();
77        }, true);
78
79        if (cleanup) {
80          EventManager.Cleanup();
81        }
82        LogFactory.GetLogger(typeof(HiveJanitor).Namespace).Log("HiveJanitor: cleanup finished.");
83      } catch (Exception e) {
84        LogFactory.GetLogger(typeof(HiveJanitor).Namespace).Log(string.Format("HiveJanitor: The following exception occured: {0}", e.ToString()));
85      }
86    }
87
88    public void RunGenerateStatistics() {
89      try {
90        LogFactory.GetLogger(typeof(HiveJanitor).Namespace).Log("HiveJanitor: starting generate statistics.");
91        StatisticsGenerator.GenerateStatistics();
92        LogFactory.GetLogger(typeof(HiveJanitor).Namespace).Log("HiveJanitor: generate statistics finished.");
93      } catch (Exception e) {
94        LogFactory.GetLogger(typeof(HiveJanitor).Namespace).Log(string.Format("HiveJanitor: The following exception occured: {0}", e));
95      }
96    }
97  }
98}
99
100
101
Note: See TracBrowser for help on using the repository browser.