Free cookie consent management tool by TermsFeed Policy Generator

source: branches/PersistenceOverhaul/HeuristicLab.Clients.Hive.Slave/3.3/SlaveTask.cs @ 13843

Last change on this file since 13843 was 12926, checked in by jkarder, 9 years ago

#2355:

  • changed sandboxing to always use an unrestricted permission set
  • removed IsAllowedPrivileged role and according IsPrivileged code
File size: 9.5 KB
Line 
1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.IO;
24using System.Threading;
25using HeuristicLab.Clients.Hive.SlaveCore.Properties;
26using HeuristicLab.Common;
27using HeuristicLab.Core;
28using HeuristicLab.PluginInfrastructure.Sandboxing;
29
30namespace HeuristicLab.Clients.Hive.SlaveCore {
31
32  /// <summary>
33  ///  Manages a single task and it's appdomain.
34  /// </summary>
35  public class SlaveTask : MarshalByRefObject {
36    private Executor executor;
37    private AppDomain appDomain;
38    private Semaphore waitForStartBeforeKillSem;
39    private bool executorMonitoringRun;
40    private Thread executorMonitoringThread;
41    private PluginManager pluginManager;
42    private ILog log;
43    public Guid TaskId { get; private set; }
44    public bool IsPrepared { get; private set; }
45    private TaskData originalTaskData;
46
47    private int coresNeeded;
48    public int CoresNeeded {
49      get { return coresNeeded; }
50      set { this.coresNeeded = value; }
51    }
52
53    public TimeSpan ExecutionTime {
54      get {
55        try {
56          return executor != null ? executor.ExecutionTime : TimeSpan.Zero;
57        }
58        catch (Exception ex) {
59          EventLogManager.LogException(ex);
60          return TimeSpan.Zero;
61        }
62      }
63    }
64
65    public SlaveTask(PluginManager pluginManager, int coresNeeded, ILog log) {
66      this.pluginManager = pluginManager;
67      this.coresNeeded = coresNeeded;
68      this.log = log;
69      waitForStartBeforeKillSem = new Semaphore(0, 1);
70      executorMonitoringRun = true;
71      IsPrepared = false;
72    }
73
74    public void StartJobAsync(Task task, TaskData taskData) {
75      try {
76        this.TaskId = task.Id;
77        originalTaskData = taskData;
78        Prepare(task);
79        StartTaskInAppDomain(taskData);
80      }
81      catch (Exception) {
82        // make sure to clean up if something went wrong
83        DisposeAppDomain();
84        throw;
85      }
86    }
87
88    public void PauseTask() {
89      if (!IsPrepared) throw new AppDomainNotCreatedException();
90      if (!executor.IsPausing && !executor.IsStopping) executor.Pause();
91    }
92
93    public void StopTask() {
94      if (!IsPrepared) throw new AppDomainNotCreatedException();
95      if (!executor.IsPausing && !executor.IsStopping) executor.Stop();
96    }
97
98    private void Prepare(Task task) {
99      string pluginDir = Path.Combine(pluginManager.PluginTempBaseDir, task.Id.ToString());
100      string configFileName;
101      pluginManager.PreparePlugins(task, out configFileName);
102      appDomain = CreateAppDomain(task, pluginDir, configFileName);
103      IsPrepared = true;
104    }
105
106    private AppDomain CreateAppDomain(Task task, String pluginDir, string configFileName) {
107      appDomain = SandboxManager.CreateAndInitSandbox(task.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
108      appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
109
110      log.LogMessage("Creating AppDomain");
111      executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
112
113      executor.TaskId = task.Id;
114      executor.CoresNeeded = task.CoresNeeded;
115      executor.MemoryNeeded = task.MemoryNeeded;
116      return appDomain;
117    }
118
119    private void StartTaskInAppDomain(TaskData taskData) {
120      executor.Start(taskData.Data);
121      waitForStartBeforeKillSem.Release();
122      StartExecutorMonitoringThread();
123    }
124
125    public void DisposeAppDomain() {
126      log.LogMessage(string.Format("Shutting down Appdomain for Task {0}", TaskId));
127      StopExecutorMonitoringThread();
128
129      if (executor != null) {
130        try {
131          executor.Dispose();
132        }
133        catch (Exception ex) {
134          EventLogManager.LogException(ex);
135        }
136      }
137
138      if (appDomain != null) {
139        appDomain.UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
140        int repeat = Settings.Default.PluginDeletionRetries;
141        while (repeat > 0) {
142          try {
143            waitForStartBeforeKillSem.WaitOne(Settings.Default.ExecutorSemTimeouts);
144            AppDomain.Unload(appDomain);
145            waitForStartBeforeKillSem.Dispose();
146            repeat = 0;
147          }
148          catch (CannotUnloadAppDomainException) {
149            log.LogMessage("Could not unload AppDomain, will try again in 1 sec.");
150            Thread.Sleep(Settings.Default.PluginDeletionTimeout);
151            repeat--;
152            if (repeat == 0) {
153              throw; // rethrow and let app crash
154            }
155          }
156        }
157      }
158      pluginManager.DeletePluginsForJob(TaskId);
159      GC.Collect();
160    }
161
162    private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
163      DisposeAppDomain();
164      OnTaskFailed(new Exception("Unhandled exception: " + e.ExceptionObject.ToString()));
165    }
166
167    public Tuple<TaskData, DateTime> GetTaskDataSnapshot() {
168      Tuple<TaskData, DateTime> snapshot = null;
169      try {
170        snapshot = executor.GetTaskDataSnapshot();
171        if (snapshot == null) return Tuple.Create(originalTaskData, DateTime.Now);
172      }
173      catch (Exception ex) {
174        EventLogManager.LogException(ex);
175      }
176      return snapshot;
177    }
178
179    public TaskData GetTaskData() {
180      TaskData data = null;
181      try {
182        data = executor.GetTaskData();
183        //this means that there was a problem executing the task
184        if (data == null) return originalTaskData;
185      }
186      catch (Exception ex) {
187        EventLogManager.LogException(ex);
188      }
189      return data;
190    }
191
192    #region ExecutorMonitorThread
193    private void StartExecutorMonitoringThread() {
194      executorMonitoringThread = new Thread(MonitorExecutor);
195      executorMonitoringThread.Start();
196    }
197
198    private void StopExecutorMonitoringThread() {
199      if (executorMonitoringThread != null) {
200        if (executorMonitoringRun) {
201          executorMonitoringRun = false;
202          executor.ExecutorCommandQueue.AddMessage(ExecutorMessageType.StopExecutorMonitoringThread);
203        }
204      }
205    }
206
207    /// <summary>
208    /// Because the executor is in an appdomain and is not able to call back
209    /// (because of security -> lease time for marshall-by-ref object is 5 min),
210    /// we have to poll the executor for events we have to react to (e.g. task finished...)   
211    /// </summary>
212    private void MonitorExecutor() {
213      while (executorMonitoringRun) {
214        //this call goes through the appdomain border. We have to
215        //poll so that the the lease gets renewed
216        ExecutorMessage message;
217        do {
218          message = executor.ExecutorCommandQueue.GetMessage();
219        } while (message == null);
220
221        switch (message.MessageType) {
222          case ExecutorMessageType.TaskStarted:
223            OnTaskStarted();
224            break;
225
226          case ExecutorMessageType.TaskPaused:
227            executorMonitoringRun = false;
228            OnTaskPaused();
229            DisposeAppDomain();
230            break;
231
232          case ExecutorMessageType.TaskStopped:
233            executorMonitoringRun = false;
234            OnTaskStopped();
235            DisposeAppDomain();
236            break;
237
238          case ExecutorMessageType.TaskFailed:
239            executorMonitoringRun = false;
240            OnTaskFailed(new TaskFailedException(executor.CurrentExceptionStr));
241            DisposeAppDomain();
242            break;
243
244          case ExecutorMessageType.StopExecutorMonitoringThread:
245            executorMonitoringRun = false;
246            break;
247        }
248      }
249    }
250    #endregion
251
252    public event EventHandler<EventArgs<Guid>> TaskStarted;
253    private void OnTaskStarted() {
254      var handler = TaskStarted;
255      if (handler != null) handler(this, new EventArgs<Guid>(this.TaskId));
256    }
257
258    public event EventHandler<EventArgs<Guid>> TaskStopped;
259    private void OnTaskStopped() {
260      var handler = TaskStopped;
261      if (handler != null) handler(this, new EventArgs<Guid>(this.TaskId));
262    }
263
264    public event EventHandler<EventArgs<Guid>> TaskPaused;
265    private void OnTaskPaused() {
266      var handler = TaskPaused;
267      if (handler != null) handler(this, new EventArgs<Guid>(this.TaskId));
268    }
269
270    public event EventHandler<EventArgs<Guid>> TaskAborted;
271    private void OnTaskAborted() {
272      var handler = TaskAborted;
273      if (handler != null) handler(this, new EventArgs<Guid>(this.TaskId));
274    }
275
276    public event EventHandler<EventArgs<Guid, Exception>> TaskFailed;
277    private void OnTaskFailed(Exception exception) {
278      var handler = TaskFailed;
279      if (handler != null) handler(this, new EventArgs<Guid, Exception>(this.TaskId, exception));
280    }
281  }
282}
Note: See TracBrowser for help on using the repository browser.