Free cookie consent management tool by TermsFeed Policy Generator

Changeset 6110 for branches


Ignore:
Timestamp:
05/03/11 17:08:54 (14 years ago)
Author:
cneumuel
Message:

#1233

  • renamed engines to executors
  • changed locking in StartJobInAppDomain
  • avoid destruction of proxy object after 5 minutes for Slave.Core
  • added JobStarted event and fixed ExecutionStateChanged and ExecutionTimeChanged
  • slaves which are moved to another slavegroup will pause their jobs now, if they must not calculate them
Location:
branches/HeuristicLab.Hive-3.4/sources
Files:
13 edited

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Tests/Mocks/MockJob.cs

    r5599 r6110  
    122122          Stopwatch watch = new Stopwatch();
    123123          watch.Start();
     124          OnJobStarted();
    124125          do {
    125126            Thread.SpinWait(1000);
    … …  
    191192    }
    192193
     194    public event EventHandler JobStarted;
     195    protected virtual void OnJobStarted() {
     196      EventHandler handler = JobStarted;
     197      if (handler != null) handler(this, EventArgs.Empty);
     198    }
     199
    193200    public event EventHandler JobFailed;
    194201    protected virtual void OnJobFailed(Exception e) {
    … …  
    210217    }
    211218    #endregion
     219
    212220  }
    213221}

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ConfigManager.cs

    r6004 r6110  
    8585      st.JobsFetched = SlaveStatusInfo.JobsFetched;
    8686
    87       Dictionary<Guid, Executor> engines = Core.ExecutionEngines;
     87      Dictionary<Guid, Executor> engines = Core.Executors;
    8888      st.Jobs = new List<JobStatus>();
    8989
    … …  
    9999    public Dictionary<Guid, TimeSpan> GetExecutionTimeOfAllJobs() {
    100100      Dictionary<Guid, TimeSpan> prog = new Dictionary<Guid, TimeSpan>();
    101       Dictionary<Guid, Executor> engines = Core.ExecutionEngines;
     101      Dictionary<Guid, Executor> engines = Core.Executors;
    102102      lock (engines) {
    103103        foreach (KeyValuePair<Guid, Executor> kvp in engines) {
    … …  
    113113
    114114    public int GetUsedCores() {
    115       Dictionary<Guid, Executor> engines = Core.ExecutionEngines;
     115      Dictionary<Guid, Executor> engines = Core.Executors;
    116116      Dictionary<Guid, Job> jobs = Core.Jobs;
    117117      int usedCores = 0;

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs

    r6107 r6110  
    4444    public static ILog Log { get; set; }
    4545
    46     private Dictionary<Guid, Executor> engines = new Dictionary<Guid, Executor>();
     46    private Dictionary<Guid, Executor> executors = new Dictionary<Guid, Executor>();
    4747    private Dictionary<Guid, AppDomain> appDomains = new Dictionary<Guid, AppDomain>();
    4848    private Dictionary<Guid, Job> jobs = new Dictionary<Guid, Job>();
    … …  
    5555    private ServiceHost slaveComm;
    5656
    57     public Dictionary<Guid, Executor> ExecutionEngines {
    58       get { return engines; }
     57    public Dictionary<Guid, Executor> Executors {
     58      get { return executors; }
    5959    }
    6060
    … …  
    162162              Job job = wcfService.GetJob(jobId);
    163163              if (job == null) throw new JobNotFoundException(jobId);
    164               lock (engines) {
     164              lock (executors) {
    165165                if (!jobs.ContainsKey(job.Id)) {
    166166                  jobs.Add(job.Id, job);
    … …  
    177177              // handle exception of task
    178178              clientCom.LogMessage(t.Exception.ToString());
    179               wcfService.UpdateJobState(container.JobId, JobState.Failed, t.Exception.ToString());
    180               SlaveStatusInfo.JobsAborted++;
    181179            }, TaskContinuationOptions.OnlyOnFaulted);
    182180            break;
    … …  
    225223
    226224        if (job != null) {
    227           engines[job.Id].Pause();
    228           JobData sJob = engines[job.Id].GetPausedJob();
    229           job.ExecutionTime = engines[job.Id].ExecutionTime;
     225          executors[job.Id].Pause();
     226          JobData sJob = executors[job.Id].GetPausedJob();
     227          job.ExecutionTime = executors[job.Id].ExecutionTime;
    230228
    231229          try {
    232             if (engines[job.Id].CurrentException != string.Empty) {
    233               wcfService.UpdateJobState(job.Id, JobState.Failed, engines[job.Id].CurrentException);
     230            if (executors[job.Id].CurrentException != string.Empty) {
     231              wcfService.UpdateJobState(job.Id, JobState.Failed, executors[job.Id].CurrentException);
    234232              SlaveStatusInfo.JobsAborted++;
    235233            } else {
    … …  
    256254
    257255        if (job != null) {
    258           engines[job.Id].Stop();
    259           JobData sJob = engines[job.Id].GetFinishedJob();
    260           job.ExecutionTime = engines[job.Id].ExecutionTime;
     256          executors[job.Id].Stop();
     257          JobData sJob = executors[job.Id].GetFinishedJob();
     258          job.ExecutionTime = executors[job.Id].ExecutionTime;
    261259
    262260
    263261          try {
    264             if (engines[job.Id].CurrentException != string.Empty) {
    265               wcfService.UpdateJobState(job.Id, JobState.Failed, engines[job.Id].CurrentException);
     262            if (executors[job.Id].CurrentException != string.Empty) {
     263              wcfService.UpdateJobState(job.Id, JobState.Failed, executors[job.Id].CurrentException);
    266264            }
    267265            SlaveStatusInfo.JobsAborted++;
    … …  
    350348
    351349
    352       lock (engines) {
    353         clientCom.LogMessage("engines locked");
     350      lock (executors) {
     351        clientCom.LogMessage("executors locked");
    354352        foreach (KeyValuePair<Guid, AppDomain> kvp in appDomains) {
    355353          clientCom.LogMessage("Shutting down Appdomain for " + kvp.Key);
    … …  
    411409      try {
    412410        clientCom.LogMessage("Getting the finished job with id: " + jobId);
    413         if (!engines.ContainsKey(jobId)) {
    414           clientCom.LogMessage("Engine doesn't exist");
     411        if (!executors.ContainsKey(jobId)) {
     412          clientCom.LogMessage("Executor doesn't exist");
    415413          return;
    416414        }
    … …  
    420418        }
    421419        Job cJob = jobs[jobId];
    422         cJob.ExecutionTime = engines[jobId].ExecutionTime;
    423 
    424         if (engines[jobId].Aborted) {
     420        cJob.ExecutionTime = executors[jobId].ExecutionTime;
     421
     422        if (executors[jobId].Aborted) {
    425423          SlaveStatusInfo.JobsAborted++;
    426424        } else {
    … …  
    428426        }
    429427
    430         if (engines[jobId].CurrentException != string.Empty) {
    431           wcfService.UpdateJobState(jobId, JobState.Failed, engines[jobId].CurrentException);
    432         }
    433 
    434         JobData sJob = engines[jobId].GetFinishedJob();
     428        if (executors[jobId].CurrentException != string.Empty) {
     429          wcfService.UpdateJobState(jobId, JobState.Failed, executors[jobId].CurrentException);
     430        }
     431
     432        JobData sJob = executors[jobId].GetFinishedJob();
    435433        try {
    436434          clientCom.LogMessage("Sending the finished job with id: " + jobId);
    … …  
    450448    }
    451449
    452     private static object locker = new object();
     450    private static object startInAppDomainLocker = new object();
    453451
    454452    /// <summary>
    … …  
    459457      clientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
    460458
    461       lock (locker) {
    462         if (engines.ContainsKey(myJob.Id)) {
     459      lock (startInAppDomainLocker) {
     460        if (executors.ContainsKey(myJob.Id)) {
    463461          clientCom.LogMessage("Job with key " + myJob.Id + " already exists. Job will be ignored.");
    464462          return;
    … …  
    476474        catch (Exception exception) {
    477475          clientCom.LogMessage(string.Format("Copying plugins for job {0} failed: {1}", myJob.Id, exception));
    478           wcfService.UpdateJobState(myJob.Id, JobState.Failed, exception.ToString());
    479           SlaveStatusInfo.JobsAborted++;
    480 
    481           lock (engines) {
     476          lock (executors) {
    482477            if (jobs.ContainsKey(myJob.Id)) {
    483478              jobs.Remove(myJob.Id);
    … …  
    490485            AppDomain appDomain = HeuristicLab.PluginInfrastructure.Sandboxing.SandboxManager.CreateAndInitSandbox(myJob.Id.ToString(), pluginDir, Path.Combine(pluginDir, configFileName));
    491486            appDomain.UnhandledException += new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
    492             Executor engine;
    493             lock (engines) {
    494               appDomains.Add(myJob.Id, appDomain);
    495               clientCom.LogMessage("Creating AppDomain");
    496               engine = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
    497               clientCom.LogMessage("Created AppDomain");
    498               engine.JobId = myJob.Id;
    499               engine.Core = this;
    500               clientCom.LogMessage("Starting Engine for job " + myJob.Id);
    501               engines.Add(myJob.Id, engine);
     487            Executor executor;
     488            appDomains.Add(myJob.Id, appDomain);
     489            clientCom.LogMessage("Creating AppDomain");
     490            executor = (Executor)appDomain.CreateInstanceAndUnwrap(typeof(Executor).Assembly.GetName().Name, typeof(Executor).FullName);
     491            clientCom.LogMessage("Created AppDomain");
     492            executor.JobId = myJob.Id;
     493            executor.Core = this;
     494            clientCom.LogMessage("Starting Executor for job " + myJob.Id);
     495
     496            executor.Start(jobData.Data);
     497           
     498            lock (executors) {
     499              executors.Add(myJob.Id, executor);
    502500            }
    503             engine.Start(jobData.Data);
    504501          }
    505502          catch (Exception exception) {
    506503            clientCom.LogMessage("Creating the Appdomain and loading the job failed for job " + myJob.Id);
    507504            clientCom.LogMessage("Error thrown is: " + exception.ToString());
    508 
    509             if (engines.ContainsKey(myJob.Id) && engines[myJob.Id].CurrentException != string.Empty) {
    510               wcfService.UpdateJobState(myJob.Id, JobState.Failed, engines[myJob.Id].CurrentException);
    511             } else {
    512               wcfService.UpdateJobState(myJob.Id, JobState.Failed, exception.ToString());
    513             }
    514             SlaveStatusInfo.JobsAborted++;
    515 
    516505            KillAppDomain(myJob.Id);
    517506          }
    … …  
    557546
    558547      clientCom.LogMessage("Shutting down Appdomain for Job " + id);
    559       lock (engines) {
     548      lock (executors) {
    560549        try {
    561           if (engines.ContainsKey(id)) {
    562             engines[id].Dispose();
    563             engines.Remove(id);
     550          if (executors.ContainsKey(id)) {
     551            executors[id].Dispose();
     552            executors.Remove(id);
    564553          }
    565554
    … …  
    597586      clientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
    598587    }
     588
     589    public override object InitializeLifetimeService() {
     590      return null; // avoid destruction of proxy object after 5 minutes
     591    }
    599592  }
    600593}

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs

    r6100 r6110  
    8585          Job.Resume(childjobs.Select(j => PersistenceUtil.Deserialize<IJob>(j.Data)));
    8686        } else {
    87           // Job.Prepare(); // do NOT prepare here, otherwise paused jobs get restarted
    8887          Job.Start();
     88
    8989        }
    9090      }
    … …  
    200200        Aborted = true;
    201201      } else {
    202         //it's a clean and finished job, so send it       
     202        //it's a clean and finished job, so send it
    203203        Core.EnqueueExecutorMessage(Core.SendFinishedJob, JobId);
    204204      }

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/RefreshableHiveExperiment.cs

    r6033 r6110  
    2525using System.Drawing;
    2626using System.Linq;
    27 using HeuristicLab.Clients.Hive.ExperimentManager;
    28 using HeuristicLab.Clients.Hive.Jobs;
    2927using HeuristicLab.Common;
    3028using HeuristicLab.Core;
     29using HeuristicLab.Hive;
    3130using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
    3231
    … …  
    3534  public class RefreshableHiveExperiment : IHiveItem, IDeepCloneable, IContent, IProgressReporter {
    3635    private JobResultPoller jobResultPoller;
     36    private JobDownloader<ItemJob> jobDownloader = new JobDownloader<ItemJob>(2, 2);
    3737
    3838    [Storable]
    … …  
    6969          refreshAutomatically = value;
    7070          OnRefreshAutomaticallyChanged();
    71           if (RefreshAutomatically) {
    72             StartResultPolling();
    73           } else {
    74             StopResultPolling();
    75           }
     71        }
     72        if (RefreshAutomatically && hiveExperiment.HiveJobs != null && hiveExperiment.HiveJobs.Count > 0 && (jobResultPoller == null || !jobResultPoller.IsPolling)) {
     73          StartResultPolling();
     74        } else {
     75          StopResultPolling();
    7676        }
    7777      }
    … …  
    103103    #endregion
    104104
    105     //public Experiment GetExperiment(int idx) {
    106     //  if (hiveExperiment.HiveJobs != null) {
    107     //    var hj = hiveExperiment.HiveJobs.ElementAtOrDefault(idx);
    108     //    if (hj != null)
    109     //      return ((OptimizerHiveJob)hj).JobItem.OptimizerAsExperiment;
    110     //  }
    111     //  return null;
    112     //}
    113 
    114     //public void AddExperiment(Experiment experiment) {
    115     //  if (hiveExperiment.HiveJobs == null)
    116     //    hiveExperiment.HiveJobs = new ItemCollection<HiveJob>();
    117     //  hiveExperiment.HiveJobs.Add(new OptimizerHiveJob(experiment));
    118     //}
    119 
    120     //public void SetExperiment(Experiment experiment) {
    121     //  if (hiveExperiment.HiveJobs == null)
    122     //    hiveExperiment.HiveJobs = new ItemCollection<HiveJob>();
    123     //  else
    124     //    hiveExperiment.HiveJobs.Clear();
    125     //  hiveExperiment.HiveJobs.Add(new OptimizerHiveJob(experiment));
    126     //}
    127 
    128105    private void hiveExperiment_HiveJobsChanged(object sender, EventArgs e) {
    129106      if (jobResultPoller != null && jobResultPoller.IsPolling) {
    … …  
    141118    public void StartResultPolling() {
    142119      if (jobResultPoller == null) {
    143         jobResultPoller = new JobResultPoller(hiveExperiment.HiveJobs, /*ApplicationConstants.ResultPollingInterval*/new TimeSpan(0, 0, 5)); //TODO: find a better place for ApplicationConstants
     120        jobResultPoller = new JobResultPoller(hiveExperiment.Id, /*ApplicationConstants.ResultPollingInterval*/new TimeSpan(0, 0, 5)); //TODO: find a better place for ApplicationConstants
    144121        RegisterResultPollingEvents();
    145122      }
    … …  
    172149    private void jobResultPoller_JobResultReceived(object sender, EventArgs<IEnumerable<LightweightJob>> e) {
    173150      foreach (LightweightJob lightweightJob in e.Value) {
    174         OptimizerHiveJob hj = GetHiveJobById(lightweightJob.Id);
     151        HiveJob hj = GetHiveJobById(lightweightJob.Id);
    175152        if (hj != null) {
    176153          DateTime lastJobDataUpdate = hj.Job.LastJobDataUpdate;
    … …  
    179156          // lastJobDataUpdate equals DateTime.MinValue right after it was uploaded. When the first results are polled, this value is updated
    180157          if (lastJobDataUpdate != DateTime.MinValue && lastJobDataUpdate < hj.Job.LastJobDataUpdate) {
    181             OptimizerJob optimizerJob = ExperimentManagerClient.LoadOptimizerJob(hj.Job.Id);
    182             if (optimizerJob == null) {
    183               // something bad happened to this job. bad job, BAAAD job!
    184             } else {
    185               // if the job is paused, download but don't integrate into parent optimizer (to avoid Prepare)
    186               if (hj.Job.State == JobState.Paused) {
    187                 hj.JobItem = optimizerJob;
     158            jobDownloader.DownloadJob(hj.Job.Id, (itemJob) => {
     159              if (itemJob == null) {
     160                // something bad happened to this job. bad job, BAAAD job!
    188161              } else {
    189                 if (lightweightJob.ParentJobId.HasValue) {
    190                   OptimizerHiveJob parentHiveJob = GetHiveJobById(lightweightJob.ParentJobId.Value);
    191                   parentHiveJob.UpdateChildOptimizer(optimizerJob, hj.Job.Id);
     162                // if the job is paused, download but don't integrate into parent optimizer (to avoid Prepare)
     163                if (hj.Job.State == JobState.Paused) {
     164                  hj.JobItem = itemJob;
     165                } else {
     166                  if (lightweightJob.ParentJobId.HasValue) {
     167                    HiveJob parentHiveJob = GetHiveJobById(lightweightJob.ParentJobId.Value);
     168                    parentHiveJob.IntegrateChild(itemJob, hj.Job.Id);
     169                  } else {
     170                    hj.JobItem = itemJob;
     171                  }
    192172                }
    193173              }
    194             }
     174            });
    195175          }
    196176        }
    … …  
    205185    }
    206186
    207     public OptimizerHiveJob GetHiveJobById(Guid jobId) {
    208       foreach (OptimizerHiveJob job in hiveExperiment.HiveJobs) {
    209         var hj = job.GetHiveJobByJobId(jobId) as OptimizerHiveJob;
     187    public HiveJob GetHiveJobById(Guid jobId) {
     188      foreach (HiveJob job in hiveExperiment.HiveJobs) {
     189        var hj = job.GetHiveJobByJobId(jobId);
    210190        if (hj != null)
    211191          return hj;
    … …  
    226206                                            || j.Job.State == JobState.Failed);
    227207    }
     208
     209    //public bool AllJobsFinishedAndDownloaded() {
     210    //  return this.AllJobsFinished() && hiveExperiment.GetAllHiveJobs().All(j => j.JobItem.;
     211    //}
    228212
    229213    private void jobResultPoller_ExceptionOccured(object sender, EventArgs<Exception> e) {
    … …  
    261245      hiveExperiment.IsProgressingChanged -= new EventHandler(hiveExperiment_IsProgressingChanged);
    262246    }
    263    
     247
    264248    #region Events
    265249    public event EventHandler RefreshAutomaticallyChanged;
    … …  
    314298    }
    315299    public void Store() {
    316       hiveExperiment.Store() ;
     300      hiveExperiment.Store();
    317301    }
    318302    public string ItemDescription {

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ServiceClients/Heartbeat.cs

    r6004 r6110  
    3131
    3232    public override string ToString() {
    33       String val = "SlaveId: " + SlaveId + ", FreeCores: " + FreeCores;
     33      string val = string.Format("SlaveId: {0}, FreeCores: {1}", SlaveId, FreeCores);
    3434      foreach (KeyValuePair<Guid, TimeSpan> kvp in JobProgress) {
    35         val += Environment.NewLine + "Id" + kvp.Key + " ExecutionTime " + kvp.Value;
     35        val += Environment.NewLine + string.Format("Id: {0}, ExecutionTime {1}", kvp.Key, kvp.Value);
    3636      }
    3737      return val;

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Hive/3.4/IJob.cs

    r5450 r6110  
    7070    event EventHandler JobPaused;
    7171
     72    event EventHandler JobStarted;
     73
    7274    /// <summary>
    7375    /// When this event occurs the job wants to sleep until all his child jobs are finished

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Hive/3.4/ItemJob.cs

    r6033 r6110  
    122122    public abstract void Resume(IEnumerable<IJob> childJobs);
    123123
     124    public event EventHandler JobStarted;
     125    protected virtual void OnJobStarted() {
     126      EventHandler handler = JobStarted;
     127      if (handler != null) handler(this, EventArgs.Empty);
     128    }
     129
    124130    public event EventHandler JobStopped;
    125131    protected virtual void OnJobStopped() {

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine.Test/Program.cs

    r6039 r6110  
    66using HeuristicLab.Algorithms.GeneticAlgorithm;
    77using HeuristicLab.Clients.Hive;
     8using HeuristicLab.Clients.Hive.Jobs;
    89using HeuristicLab.Common;
    910using HeuristicLab.Core;
     11using HeuristicLab.Optimization;
    1012using HeuristicLab.PluginInfrastructure;
    1113using HeuristicLab.PluginInfrastructure.Manager;
    1214using HeuristicLab.Problems.TestFunctions;
    13 using HeuristicLab.Clients.Hive.Jobs;
    14 using HeuristicLab.Optimization;
    1515
    1616namespace HeuristicLab.HiveEngine.Test {
    … …  
    4040
    4141      var job2 = PersistenceUtil.Deserialize<OptimizerJob>(data);
    42 
    43 
    44 
     42     
    4543      #region Credentials
    4644      ServiceLocator.Instance.Username = "cneumuel";

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/HiveDao.cs

    r6006 r6110  
    546546    /// Returns all parent resources of a resource (the given resource is also added)
    547547    /// </summary>
    548     private IEnumerable<DT.Resource> GetParentResources(Guid resourceId) {
     548    public IEnumerable<DT.Resource> GetParentResources(Guid resourceId) {
    549549      using (var db = CreateContext()) {
    550550        var resources = new List<Resource>();

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/Interfaces/IHiveDao.cs

    r5636 r6110  
    9494    void AssignJobToResource(Guid jobId, Guid resourceId);
    9595    IEnumerable<DT.Resource> GetAssignedResources(Guid jobId);
     96    IEnumerable<DT.Resource> GetParentResources(Guid resourceId);
    9697    #endregion
    9798

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/HeartbeatManager.cs

    r5786 r6110  
    8282            actions.Add(new MessageContainer(MessageContainer.MessageType.AbortJob, curJob.Id));
    8383            LogFactory.GetLogger(this.GetType().Namespace).Log("The slave " + heartbeat.SlaveId + " is not supposed to calculate Job: " + curJob);
     84          } else if (!JobIsAllowedToBeCalculatedBySlave(heartbeat.SlaveId, curJob)) {
     85            // assigned resources ids of job do not match with slaveId (and parent resourceGroupIds); this might happen when slave is moved to different group
     86            actions.Add(new MessageContainer(MessageContainer.MessageType.PauseJob, curJob.Id));
    8487          } else {
    8588            // save job execution time
    … …  
    104107      return actions;
    105108    }
     109
     110    private bool JobIsAllowedToBeCalculatedBySlave(Guid slaveId, Job curJob) {
     111      var assignedResourceIds = dao.GetAssignedResources(curJob.Id).Select(x => x.Id);
     112      var slaveResourceIds = dao.GetParentResources(slaveId).Select(x => x.Id);
     113      return assignedResourceIds.Any(x => slaveResourceIds.Contains(x));
     114    }
    106115  }
    107116}

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/HiveService.cs

    r6006 r6110  
    147147      return trans.UseTransaction(() => {
    148148        Job job = dao.UpdateJobState(jobId, jobState, slaveId, userId, exception);
     149       
    149150        if (job.Command.HasValue && job.Command.Value == Command.Pause && job.State == JobState.Paused) {
    150151          job.Command = null;
    … …  
    153154        } else if (job.Command.HasValue && job.Command.Value == Command.Stop && job.State == JobState.Aborted) {
    154155          job.Command = null;
    155         }
     156        } else if (jobState == JobState.Paused && !job.Command.HasValue) {
     157          // job was paused and uploaded by slave without the user-command to pause it -> set waiting
     158          job = dao.UpdateJobState(jobId, JobState.Waiting, slaveId, userId, exception);
     159        }
     160
    156161        dao.UpdateJob(job);
    157162        return job;
Note: See TracChangeset for help on using the changeset viewer.