Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveJob.cs @ 6456

Last change on this file since 6456 was 6456, checked in by ascheibe, 12 years ago

#1233

  • fixed Admin Views plugin dependencies
  • use settings instead of magic numbers and strings
File size: 8.8 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2011 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.IO;
24using System.Threading;
25using HeuristicLab.Clients.Hive.SlaveCore.Properties;
26using HeuristicLab.Common;
27using HeuristicLab.Core;
28using HeuristicLab.PluginInfrastructure.Sandboxing;
29
30namespace HeuristicLab.Clients.Hive.SlaveCore {
31
32  /// <summary>
33  ///  Manages a single job and it's appdomain.
34  /// </summary>
35  public class SlaveJob : MarshalByRefObject {
36    private Executor executor;
37    private AppDomain appDomain;
38    private Semaphore waitForStartBeforeKillSem;
39    private bool executorMonitoringRun;
40    private Thread executorMonitoringThread;
41    private PluginManager pluginManager;
42    private ILog log;
43    public Guid JobId { get; private set; }
44    public bool IsPrepared { get; private set; }
45
46    private int coresNeeded;
47    public int CoresNeeded {
48      get { return coresNeeded; }
49      set { this.coresNeeded = value; }
50    }
51
52    public TimeSpan ExecutionTime {
53      get { return executor != null ? executor.ExecutionTime : TimeSpan.Zero; }
54    }
55
56    public SlaveJob(PluginManager pluginManager, int coresNeeded, ILog log) {
57      this.pluginManager = pluginManager;
58      this.coresNeeded = coresNeeded;
59      this.log = log;
60      waitForStartBeforeKillSem = new Semaphore(0, 1);
61      executorMonitoringRun = true;
62      IsPrepared = false;
63    }
64
65    public void StartJobAsync(Job job, JobData jobData) {
66      try {
67        this.JobId = job.Id;
68        Prepare(job);
69        StartJobInAppDomain(jobData);
70      }
71      catch (Exception) {
72        // make sure to clean up if something went wrong
73        DisposeAppDomain();
74        throw;
75      }
76    }
77
78    public void PauseJob() {
79      if (!IsPrepared) throw new AppDomainNotCreatedException();
80      if (!executor.IsPausing && !executor.IsStopping) executor.Pause();
81    }
82
83    public void StopJob() {
84      if (!IsPrepared) throw new AppDomainNotCreatedException();
85      if (!executor.IsPausing && !executor.IsStopping) executor.Stop();
86    }
87
88    private void Prepare(Job job) {
89      string pluginDir = Path.Combine(pluginManager.PluginTempBaseDir, job.Id.ToString());
90      string configFileName;
91      pluginManager.PreparePlugins(job, out configFileName);
92      appDomain = CreateAppDomain(job, pluginDir, configFileName);
93      IsPrepared = true;
94    }
95
96    private AppDomain CreateAppDomain(Job job, String pluginDir, string configFileName) {
97      if (job.IsPrivileged) {
98        appDomain = SandboxManager.CreateAndInitPrivilegedSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
99      } else {
100        appDomain = SandboxManager.CreateAndInitSandbox(job.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
101      }
102      appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
103
104      log.LogMessage("Creating AppDomain");
105      executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
106
107      executor.JobId = job.Id;
108      executor.CoresNeeded = job.CoresNeeded;
109      executor.MemoryNeeded = job.MemoryNeeded;
110      return appDomain;
111    }
112
113    private void StartJobInAppDomain(JobData jobData) {
114      executor.Start(jobData.Data);
115      waitForStartBeforeKillSem.Release();
116      StartExecutorMonitoringThread();
117    }
118
119    public void DisposeAppDomain() {
120      log.LogMessage(string.Format("Shutting down Appdomain for Job {0}", JobId));
121      StopExecutorMonitoringThread();
122
123      if (executor != null) {
124        executor.Dispose();
125      }
126
127      if (appDomain != null) {
128        appDomain.UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
129        int repeat = Settings.Default.PluginDeletionRetries;
130        while (repeat > 0) {
131          try {
132            waitForStartBeforeKillSem.WaitOne();
133            AppDomain.Unload(appDomain);
134            waitForStartBeforeKillSem.Dispose();
135            repeat = 0;
136          }
137          catch (CannotUnloadAppDomainException) {
138            log.LogMessage("Could not unload AppDomain, will try again in 1 sec.");
139            Thread.Sleep(Settings.Default.PluginDeletionTimeout);
140            repeat--;
141            if (repeat == 0) {
142              throw; // rethrow and let app crash
143            }
144          }
145        }
146      }
147      pluginManager.DeletePluginsForJob(JobId);
148      GC.Collect();
149    }
150
151    private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
152      DisposeAppDomain();
153      OnExceptionOccured(new Exception("Unhandled exception: " + e.ExceptionObject.ToString()));
154    }
155
156    public JobData GetJobData() {
157      return executor.GetJobData();
158    }
159
160    #region ExecutorMonitorThread
161    private void StartExecutorMonitoringThread() {
162      executorMonitoringThread = new Thread(MonitorExecutor);
163      executorMonitoringThread.Start();
164    }
165
166    private void StopExecutorMonitoringThread() {
167      if (executorMonitoringThread != null) {
168        if (executorMonitoringRun) {
169          executorMonitoringRun = false;
170          executor.ExecutorCommandQueue.AddMessage(ExecutorMessageType.StopExecutorMonitoringThread);
171        }
172      }
173    }
174
175    /// <summary>
176    /// Because the executor is in an appdomain and is not able to call back
177    /// (because of security -> lease time for marshall-by-ref object is 5 min),
178    /// we have to poll the executor for events we have to react to (e.g. job finished...)   
179    /// </summary>
180    private void MonitorExecutor() {
181      while (executorMonitoringRun) {
182        //this blocks through the appdomain border, that's why the lease gets renewed
183        ExecutorMessage message = executor.ExecutorCommandQueue.GetMessage();
184
185        switch (message.MessageType) {
186          case ExecutorMessageType.JobStarted:
187            OnJobStarted();
188            break;
189
190          case ExecutorMessageType.JobPaused:
191            executorMonitoringRun = false;
192            OnJobPaused();
193            DisposeAppDomain();
194            break;
195
196          case ExecutorMessageType.JobStopped:
197            executorMonitoringRun = false;
198            OnJobStopped();
199            DisposeAppDomain();
200            break;
201
202          case ExecutorMessageType.JobFailed:
203            executorMonitoringRun = false;
204            OnJobFailed(new JobFailedException(executor.CurrentException));
205            DisposeAppDomain();
206            break;
207
208          case ExecutorMessageType.StopExecutorMonitoringThread:
209            executorMonitoringRun = false;
210            return;
211        }
212      }
213    }
214    #endregion
215
216    public event EventHandler<EventArgs<Guid>> JobStarted;
217    private void OnJobStarted() {
218      var handler = JobStarted;
219      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
220    }
221
222    public event EventHandler<EventArgs<Guid>> JobStopped;
223    private void OnJobStopped() {
224      var handler = JobStopped;
225      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
226    }
227
228    public event EventHandler<EventArgs<Guid>> JobPaused;
229    private void OnJobPaused() {
230      var handler = JobPaused;
231      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
232    }
233
234    public event EventHandler<EventArgs<Guid>> JobAborted;
235    private void OnJobAborted() {
236      var handler = JobAborted;
237      if (handler != null) handler(this, new EventArgs<Guid>(this.JobId));
238    }
239
240    public event EventHandler<EventArgs<Guid, Exception>> JobFailed;
241    private void OnJobFailed(Exception exception) {
242      var handler = JobFailed;
243      if (handler != null) handler(this, new EventArgs<Guid, Exception>(this.JobId, exception));
244    }
245
246    public event EventHandler<EventArgs<Guid, Exception>> ExceptionOccured;
247    private void OnExceptionOccured(Exception exception) {
248      var handler = ExceptionOccured;
249      if (handler != null) handler(this, new EventArgs<Guid, Exception>(this.JobId, exception));
250    }
251  }
252}
Note: See TracBrowser for help on using the repository browser.