[265] | 1 | #region License Information
|
---|
| 2 | /* HeuristicLab
|
---|
| 3 | * Copyright (C) 2002-2008 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
|
---|
| 4 | *
|
---|
| 5 | * This file is part of HeuristicLab.
|
---|
| 6 | *
|
---|
| 7 | * HeuristicLab is free software: you can redistribute it and/or modify
|
---|
| 8 | * it under the terms of the GNU General Public License as published by
|
---|
| 9 | * the Free Software Foundation, either version 3 of the License, or
|
---|
| 10 | * (at your option) any later version.
|
---|
| 11 | *
|
---|
| 12 | * HeuristicLab is distributed in the hope that it will be useful,
|
---|
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
| 15 | * GNU General Public License for more details.
|
---|
| 16 | *
|
---|
| 17 | * You should have received a copy of the GNU General Public License
|
---|
| 18 | * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
|
---|
| 19 | */
|
---|
| 20 | #endregion
|
---|
| 21 |
|
---|
| 22 | using System;
|
---|
[219] | 23 | using System.Collections.Generic;
|
---|
| 24 | using System.Linq;
|
---|
| 25 | using System.Text;
|
---|
| 26 | using System.ServiceModel;
|
---|
| 27 | using HeuristicLab.Grid;
|
---|
| 28 | using System.Threading;
|
---|
| 29 | using HeuristicLab.Core;
|
---|
| 30 | using System.IO;
|
---|
| 31 | using System.Windows.Forms;
|
---|
[386] | 32 | using System.Diagnostics;
|
---|
[219] | 33 |
|
---|
[372] | 34 | namespace HeuristicLab.Grid {
|
---|
[391] | 35 | public class JobExecutionException : ApplicationException {
|
---|
| 36 | public JobExecutionException(string msg) : base(msg) { }
|
---|
| 37 | }
|
---|
| 38 |
|
---|
[372] | 39 | public class JobManager {
|
---|
[386] | 40 | private const int MAX_RESTARTS = 5;
|
---|
| 41 | private const int MAX_CONNECTION_RETRIES = 10;
|
---|
| 42 | private const int RETRY_TIMEOUT_SEC = 60;
|
---|
[502] | 43 | private const int RESULT_POLLING_TIMEOUT = 5;
|
---|
[386] | 44 |
|
---|
[391] | 45 | private class Job {
|
---|
| 46 | public Guid guid;
|
---|
| 47 | public ProcessingEngine engine;
|
---|
| 48 | public ManualResetEvent waitHandle;
|
---|
| 49 | public int restarts;
|
---|
| 50 | }
|
---|
| 51 |
|
---|
[219] | 52 | private IGridServer server;
|
---|
| 53 | private string address;
|
---|
[386] | 54 | private object waitingQueueLock = new object();
|
---|
[391] | 55 | private Queue<Job> waitingJobs = new Queue<Job>();
|
---|
[386] | 56 | private object runningQueueLock = new object();
|
---|
[391] | 57 | private Queue<Job> runningJobs = new Queue<Job>();
|
---|
[248] | 58 | private Dictionary<AtomicOperation, byte[]> results = new Dictionary<AtomicOperation, byte[]>();
|
---|
[386] | 59 |
|
---|
[315] | 60 | private List<IOperation> erroredOperations = new List<IOperation>();
|
---|
[248] | 61 | private object connectionLock = new object();
|
---|
| 62 | private object dictionaryLock = new object();
|
---|
| 63 |
|
---|
[387] | 64 | private AutoResetEvent runningWaitHandle = new AutoResetEvent(false);
|
---|
| 65 | private AutoResetEvent waitingWaitHandle = new AutoResetEvent(false);
|
---|
[248] | 66 |
|
---|
[228] | 67 | private ChannelFactory<IGridServer> factory;
|
---|
[219] | 68 |
|
---|
| 69 | public JobManager(string address) {
|
---|
| 70 | this.address = address;
|
---|
[386] | 71 | Thread starterThread = new Thread(StartEngines);
|
---|
| 72 | Thread resultsGatheringThread = new Thread(GetResults);
|
---|
| 73 | starterThread.Start();
|
---|
| 74 | resultsGatheringThread.Start();
|
---|
[219] | 75 | }
|
---|
| 76 |
|
---|
[372] | 77 | public void Reset() {
|
---|
[248] | 78 | ResetConnection();
|
---|
| 79 | lock(dictionaryLock) {
|
---|
[391] | 80 | foreach(Job j in waitingJobs) {
|
---|
| 81 | j.waitHandle.Close();
|
---|
| 82 | }
|
---|
| 83 | waitingJobs.Clear();
|
---|
| 84 | foreach(Job j in runningJobs) {
|
---|
| 85 | j.waitHandle.Close();
|
---|
| 86 | }
|
---|
| 87 | runningJobs.Clear();
|
---|
[248] | 88 | results.Clear();
|
---|
[315] | 89 | erroredOperations.Clear();
|
---|
[219] | 90 | }
|
---|
| 91 | }
|
---|
| 92 |
|
---|
[228] | 93 | private void ResetConnection() {
|
---|
[402] | 94 | Trace.TraceInformation("Reset connection in JobManager");
|
---|
[248] | 95 | lock(connectionLock) {
|
---|
| 96 | // open a new channel
|
---|
| 97 | NetTcpBinding binding = new NetTcpBinding();
|
---|
| 98 | binding.MaxReceivedMessageSize = 100000000; // 100Mbytes
|
---|
| 99 | binding.ReaderQuotas.MaxStringContentLength = 100000000; // also 100M chars
|
---|
| 100 | binding.ReaderQuotas.MaxArrayLength = 100000000; // also 100M elements;
|
---|
| 101 | binding.Security.Mode = SecurityMode.None;
|
---|
| 102 |
|
---|
| 103 | factory = new ChannelFactory<IGridServer>(binding);
|
---|
| 104 | server = factory.CreateChannel(new EndpointAddress(address));
|
---|
| 105 | }
|
---|
[228] | 106 | }
|
---|
| 107 |
|
---|
[386] | 108 | public void StartEngines() {
|
---|
| 109 | try {
|
---|
| 110 | while(true) {
|
---|
[391] | 111 | Job job = null;
|
---|
[386] | 112 | lock(waitingQueueLock) {
|
---|
[391] | 113 | if(waitingJobs.Count > 0) job = waitingJobs.Dequeue();
|
---|
[248] | 114 | }
|
---|
[391] | 115 | if(job==null) waitingWaitHandle.WaitOne(); // no jobs waiting
|
---|
| 116 | else {
|
---|
| 117 | Guid currentEngineGuid = TryStartExecuteEngine(job.engine);
|
---|
| 118 | if(currentEngineGuid == Guid.Empty) {
|
---|
| 119 | // couldn't start the job -> requeue
|
---|
| 120 | if(job.restarts < MAX_RESTARTS) {
|
---|
| 121 | job.restarts++;
|
---|
| 122 | lock(waitingQueueLock) waitingJobs.Enqueue(job);
|
---|
| 123 | waitingWaitHandle.Set();
|
---|
[386] | 124 | } else {
|
---|
[391] | 125 | // max restart count reached -> give up on this job and flag error
|
---|
| 126 | lock(dictionaryLock) {
|
---|
| 127 | erroredOperations.Add(job.engine.InitialOperation);
|
---|
| 128 | job.waitHandle.Set();
|
---|
[386] | 129 | }
|
---|
| 130 | }
|
---|
[391] | 131 | } else {
|
---|
| 132 | // job started successfully
|
---|
| 133 | job.guid = currentEngineGuid;
|
---|
| 134 | lock(runningQueueLock) {
|
---|
| 135 | runningJobs.Enqueue(job);
|
---|
| 136 | runningWaitHandle.Set();
|
---|
| 137 | }
|
---|
[386] | 138 | }
|
---|
[315] | 139 | }
|
---|
[386] | 140 | }
|
---|
[402] | 141 | } catch(Exception e) {
|
---|
| 142 | Trace.TraceError("Exception "+e+" in JobManager.StartEngines() killed the start-engine thread\n"+e.StackTrace);
|
---|
[386] | 143 | }
|
---|
| 144 | }
|
---|
| 145 |
|
---|
[391] | 146 |
|
---|
[386] | 147 | public void GetResults() {
|
---|
| 148 | try {
|
---|
| 149 | while(true) {
|
---|
[391] | 150 | Job job = null;
|
---|
[386] | 151 | lock(runningQueueLock) {
|
---|
[391] | 152 | if(runningJobs.Count > 0) job = runningJobs.Dequeue();
|
---|
[315] | 153 | }
|
---|
[391] | 154 | if(job == null) runningWaitHandle.WaitOne(); // no jobs running
|
---|
| 155 | else {
|
---|
| 156 | byte[] zippedResult = TryEndExecuteEngine(server, job.guid);
|
---|
[386] | 157 | if(zippedResult != null) { // successful
|
---|
| 158 | lock(dictionaryLock) {
|
---|
| 159 | // store result
|
---|
[391] | 160 | results[job.engine.InitialOperation] = zippedResult;
|
---|
| 161 | // notify consumer that result is ready
|
---|
| 162 | job.waitHandle.Set();
|
---|
[386] | 163 | }
|
---|
| 164 | } else {
|
---|
| 165 | // there was a problem -> check the state of the job and restart if necessary
|
---|
[391] | 166 | JobState jobState = TryGetJobState(server, job.guid);
|
---|
| 167 | if(jobState == JobState.Unknown) {
|
---|
| 168 | job.restarts++;
|
---|
[386] | 169 | lock(waitingQueueLock) {
|
---|
[391] | 170 | waitingJobs.Enqueue(job);
|
---|
[386] | 171 | waitingWaitHandle.Set();
|
---|
| 172 | }
|
---|
| 173 | } else {
|
---|
| 174 | // job still active at the server
|
---|
| 175 | lock(runningQueueLock) {
|
---|
[391] | 176 | runningJobs.Enqueue(job);
|
---|
| 177 | runningWaitHandle.Set();
|
---|
[386] | 178 | }
|
---|
[520] | 179 | Thread.Sleep(TimeSpan.FromSeconds(RESULT_POLLING_TIMEOUT)); // sleep a while before trying to get the next result
|
---|
[386] | 180 | }
|
---|
| 181 | }
|
---|
| 182 | }
|
---|
[248] | 183 | }
|
---|
[402] | 184 | } catch(Exception e) {
|
---|
| 185 | Trace.TraceError("Exception " + e + " in JobManager.GetResults() killed the results-gathering thread\n"+ e.StackTrace);
|
---|
[219] | 186 | }
|
---|
| 187 | }
|
---|
| 188 |
|
---|
[386] | 189 | public WaitHandle BeginExecuteOperation(IScope globalScope, AtomicOperation operation) {
|
---|
[414] | 190 | return BeginExecuteEngine(new ProcessingEngine(globalScope, operation));
|
---|
| 191 | }
|
---|
| 192 |
|
---|
| 193 | public WaitHandle BeginExecuteEngine(ProcessingEngine engine) {
|
---|
[391] | 194 | Job job = new Job();
|
---|
[414] | 195 | job.engine = engine;
|
---|
[391] | 196 | job.waitHandle = new ManualResetEvent(false);
|
---|
| 197 | job.restarts = 0;
|
---|
[386] | 198 | lock(waitingQueueLock) {
|
---|
[391] | 199 | waitingJobs.Enqueue(job);
|
---|
[386] | 200 | }
|
---|
| 201 | waitingWaitHandle.Set();
|
---|
[391] | 202 | return job.waitHandle;
|
---|
[386] | 203 | }
|
---|
| 204 |
|
---|
[257] | 205 | private byte[] ZipEngine(ProcessingEngine engine) {
|
---|
[402] | 206 | return PersistenceManager.SaveToGZip(engine);
|
---|
[257] | 207 | }
|
---|
| 208 |
|
---|
[281] | 209 | public ProcessingEngine EndExecuteOperation(AtomicOperation operation) {
|
---|
[315] | 210 | if(erroredOperations.Contains(operation)) {
|
---|
| 211 | erroredOperations.Remove(operation);
|
---|
[391] | 212 | throw new JobExecutionException("Maximal number of job restarts reached. There is a problem with the connection to the grid-server.");
|
---|
[315] | 213 | } else {
|
---|
| 214 | byte[] zippedResult = null;
|
---|
| 215 | lock(dictionaryLock) {
|
---|
| 216 | zippedResult = results[operation];
|
---|
| 217 | results.Remove(operation);
|
---|
| 218 | }
|
---|
| 219 | // restore the engine
|
---|
[402] | 220 | return (ProcessingEngine)PersistenceManager.RestoreFromGZip(zippedResult);
|
---|
[256] | 221 | }
|
---|
[248] | 222 | }
|
---|
| 223 |
|
---|
[391] | 224 | private Guid TryStartExecuteEngine(ProcessingEngine engine) {
|
---|
| 225 | byte[] zippedEngine = ZipEngine(engine);
|
---|
| 226 | int retries = 0;
|
---|
| 227 | Guid guid = Guid.Empty;
|
---|
| 228 | do {
|
---|
| 229 | try {
|
---|
| 230 | lock(connectionLock) {
|
---|
| 231 | guid = server.BeginExecuteEngine(zippedEngine);
|
---|
| 232 | }
|
---|
| 233 | return guid;
|
---|
| 234 | } catch(TimeoutException) {
|
---|
| 235 | retries++;
|
---|
| 236 | Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
|
---|
| 237 | } catch(CommunicationException) {
|
---|
| 238 | ResetConnection();
|
---|
| 239 | retries++;
|
---|
| 240 | Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
|
---|
| 241 | }
|
---|
| 242 | } while(retries < MAX_CONNECTION_RETRIES);
|
---|
[402] | 243 | Trace.TraceWarning("Reached max connection retries in TryStartExecuteEngine");
|
---|
[391] | 244 | return Guid.Empty;
|
---|
| 245 | }
|
---|
| 246 |
|
---|
[315] | 247 | private byte[] TryEndExecuteEngine(IGridServer server, Guid engineGuid) {
|
---|
| 248 | int retries = 0;
|
---|
| 249 | do {
|
---|
| 250 | try {
|
---|
| 251 | lock(connectionLock) {
|
---|
[501] | 252 | byte[] zippedResult = server.TryEndExecuteEngine(engineGuid);
|
---|
[315] | 253 | return zippedResult;
|
---|
| 254 | }
|
---|
[383] | 255 | } catch(TimeoutException) {
|
---|
[315] | 256 | retries++;
|
---|
| 257 | Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
|
---|
[383] | 258 | } catch(CommunicationException) {
|
---|
[315] | 259 | ResetConnection();
|
---|
| 260 | retries++;
|
---|
| 261 | Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
|
---|
| 262 | }
|
---|
| 263 | } while(retries < MAX_CONNECTION_RETRIES);
|
---|
[402] | 264 | Trace.TraceWarning("Reached max connection retries in TryEndExecuteEngine");
|
---|
[315] | 265 | return null;
|
---|
| 266 | }
|
---|
| 267 |
|
---|
| 268 | private JobState TryGetJobState(IGridServer server, Guid engineGuid) {
|
---|
| 269 | // check if the server is still working on the job
|
---|
| 270 | int retries = 0;
|
---|
| 271 | do {
|
---|
| 272 | try {
|
---|
| 273 | lock(connectionLock) {
|
---|
| 274 | JobState jobState = server.JobState(engineGuid);
|
---|
| 275 | return jobState;
|
---|
| 276 | }
|
---|
[383] | 277 | } catch(TimeoutException) {
|
---|
[315] | 278 | retries++;
|
---|
| 279 | Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
|
---|
[383] | 280 | } catch(CommunicationException) {
|
---|
[315] | 281 | ResetConnection();
|
---|
| 282 | retries++;
|
---|
| 283 | Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
|
---|
| 284 | }
|
---|
| 285 | } while(retries < MAX_CONNECTION_RETRIES);
|
---|
[402] | 286 | Trace.TraceWarning("Reached max connection retries in TryGetJobState");
|
---|
[391] | 287 | return JobState.Unknown;
|
---|
[315] | 288 | }
|
---|
[219] | 289 | }
|
---|
| 290 | }
|
---|