Changeset 6216

Timestamp:

05/17/11 11:26:48 (14 years ago)

Author:

ascheibe

Message:

#1233

• make UsedCores more reliable
• some cosmetic fixes

Location:

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4

Files:

• ConfigManager.cs (modified) (2 diffs)
• Core.cs (modified) (9 diffs)
• SlaveJob.cs (modified) (13 diffs)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ConfigManager.cs

@@ -67 +67 @@
     }
 
+    public int GetFreeCores() {
+      return slave.Cores.HasValue ? slave.Cores.Value - SlaveStatusInfo.UsedCores : 0;
+    }
+
     /// <summary>
     /// collects and returns information that get displayed by the Client Console
@@ -79 +83 @@
 
       st.TotalCores = slave.Cores.HasValue ? slave.Cores.Value : 0;
-      st.FreeCores = slave.Cores.HasValue ? slave.Cores.Value - SlaveStatusInfo.UsedCores : 0;
+      st.FreeCores = GetFreeCores();
 
       st.JobsAborted = SlaveStatusInfo.JobsAborted;

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs

@@ -37 +37 @@
   /// </summary>
   public class Core : MarshalByRefObject {
-    public EventLog ServiceEventLog { get; set; }
-
-    public static bool abortRequested { get; set; }
+    private ISlaveCommunication clientCom;
+    private ServiceHost slaveComm;
     private Semaphore waitShutdownSem = new Semaphore(0, 1);
-    public static ILog Log { get; set; }
-
     private Dictionary<Guid, SlaveJob> slaveJobs = new Dictionary<Guid, SlaveJob>();
-
     private WcfService wcfService;
     private static HeartbeatManager heartbeatManager;
+
     public static HeartbeatManager HBManager { get { return heartbeatManager; } }
-
-    private ISlaveCommunication clientCom;
-    private ServiceHost slaveComm;
-
-    public Dictionary<Guid, SlaveJob> SlaveJobs {
-      get { return slaveJobs; }
-    }
+    public static ILog Log { get; set; }
+    public EventLog ServiceEventLog { get; set; }
+    public static bool abortRequested { get; set; }
+    public Dictionary<Guid, SlaveJob> SlaveJobs { get { return slaveJobs; } }
 
     public Core() { }
@@ -86 +80 @@
         if (ServiceEventLog != null) {
           try {
-            ServiceEventLog.WriteEntry("Hive Slave threw exception: " + ex.ToString() + " with stack trace: " + ex.StackTrace);
+            ServiceEventLog.WriteEntry(string.Format("Hive Slave threw exception: {0} with stack trace: {1}", ex.ToString(), ex.StackTrace));
           }
           catch (Exception) { }
@@ -92 +86 @@
           //try to log with clientCom. if this works the user sees at least a message, 
           //else an exception will be thrown anyways.
-          clientCom.LogMessage("Uncaught exception: " + ex.ToString() +
-            Environment.NewLine + "Core is going to shutdown.");
+          clientCom.LogMessage(string.Format("Uncaught exception: {0} {1} Core is going to shutdown.", ex.ToString(), Environment.NewLine));
         }
         ShutdownCore();
@@ -133 +126 @@
 
     void WcfService_ExceptionOccured(object sender, EventArgs<Exception> e) {
-      clientCom.LogMessage("Connection to server interruped with exception: " + e.Value.Message);
+      clientCom.LogMessage(string.Format("Connection to server interruped with exception: {0}", e.Value.Message));
     }
 
@@ -145 +138 @@
     /// <param name="container">The container, containing the message</param>
     private void DetermineAction(MessageContainer container) {
-      clientCom.LogMessage("Message: " + container.Message.ToString() + " for job: " + container.JobId);
+      clientCom.LogMessage(string.Format("Message: {0} for job: {1} ", container.Message.ToString(), container.JobId));
 
       if (container is ExecutorMessageContainer<Guid>) {
@@ -153 +146 @@
         switch (container.Message) {
           case MessageContainer.MessageType.CalculateJob:
-            Task.Factory.StartNew((jobIdObj) => {
-              Guid jobId = (Guid)jobIdObj;
-              SlaveJob newJob = new SlaveJob(this);
-              bool start = true;
-
-              lock (slaveJobs) {
-                if (slaveJobs.ContainsKey(jobId)) {
-                  start = false;
-                  clientCom.LogMessage(string.Format("Job with id {0} already exists. Start aborted.", jobId));
-                } else {
-                  slaveJobs.Add(jobId, newJob);
-                }
-              }
-
-              if (start) {
-                newJob.CalculateJob(jobId);
-              }
-            }, container.JobId)
+            Task.Factory.StartNew(HandleCalculateJob, container.JobId)
             .ContinueWith((t) => {
               // handle exception of task
@@ -193 +169 @@
             SlaveStatusInfo.IncrementJobsAborted(); //TODO: move to a sane place
 
-            Task.Factory.StartNew((jobIdObj) => {
-              Guid jobId = (Guid)jobIdObj;
-              bool abort = true;
-              SlaveJob sj = null;
-
-              lock (slaveJobs) {
-                if (!slaveJobs.ContainsKey(jobId)) {
-                  clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Abort aborted.", jobId));
-                  abort = false;
-                } else {
-                  sj = slaveJobs[jobId];
-                }
-              }
-              if (abort && !sj.Finished) {
-                sj.KillAppDomain();
-              }
-            }, container.JobId)
+            Task.Factory.StartNew(HandleAbortJob, container.JobId)
              .ContinueWith((t) => {
-               // handle exception of task
                clientCom.LogMessage(t.Exception.ToString());
              }, TaskContinuationOptions.OnlyOnFaulted);
             break;
           case MessageContainer.MessageType.StopJob:
-            Task.Factory.StartNew((jobIdObj) => {
-              Guid jobId = (Guid)jobIdObj;
-              bool stop = true;
-              SlaveJob sj = null;
-
-              lock (slaveJobs) {
-                if (!slaveJobs.ContainsKey(jobId)) {
-                  clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Stop aborted.", jobId));
-                  stop = false;
-                } else {
-                  sj = slaveJobs[jobId];
-                }
-              }
-              if (stop && !sj.Finished) {
-                sj.StopJob();
-              }
-            }, container.JobId)
+            Task.Factory.StartNew(HandleStopJob, container.JobId)
              .ContinueWith((t) => {
-               // handle exception of task
                clientCom.LogMessage(t.Exception.ToString());
              }, TaskContinuationOptions.OnlyOnFaulted);
             break;
           case MessageContainer.MessageType.PauseJob:
-            Task.Factory.StartNew((jobIdObj) => {
-              Guid jobId = (Guid)jobIdObj;
-              bool pause = true;
-              SlaveJob sj = null;
-
-              lock (slaveJobs) {
-                if (!slaveJobs.ContainsKey(jobId)) {
-                  clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Pause aborted.", jobId));
-                  pause = false;
-                } else {
-                  sj = slaveJobs[jobId];
-                }
-              }
-              if (pause && !sj.Finished) {
-                sj.PauseJob();
-              }
-            }, container.JobId)
+            Task.Factory.StartNew(HandlePauseJob, container.JobId)
              .ContinueWith((t) => {
-               // handle exception of task
                clientCom.LogMessage(t.Exception.ToString());
              }, TaskContinuationOptions.OnlyOnFaulted);
@@ -273 +198 @@
       } else {
         clientCom.LogMessage("Unknown MessageContainer: " + container);
+      }
+    }
+
+    private void HandleCalculateJob(object jobIdObj) {
+      Guid jobId = (Guid)jobIdObj;
+      SlaveJob newJob = new SlaveJob(this);
+      bool start = true;
+
+      lock (slaveJobs) {
+        if (ConfigManager.Instance.GetFreeCores() < 1) {
+          wcfService.UpdateJobState(jobId, JobState.Waiting, "Slave set status waiting because no cores were available");
+          clientCom.LogMessage(string.Format("Setting job with id {0} to waiting, all cores are used", jobId));
+          start = false;
+        } else {
+          if (slaveJobs.ContainsKey(jobId)) {
+            start = false;
+            clientCom.LogMessage(string.Format("Job with id {0} already exists. Start aborted.", jobId));
+          } else {
+            slaveJobs.Add(jobId, newJob);
+            newJob.PrepareJob(jobId);
+          }
+        }
+      }
+
+      if (start) {
+        newJob.CalculateJob();
+      }
+    }
+
+    private void HandleAbortJob(object jobIdObj) {
+      Guid jobId = (Guid)jobIdObj;
+      bool abort = true;
+      SlaveJob sj = null;
+
+      lock (slaveJobs) {
+        if (!slaveJobs.ContainsKey(jobId)) {
+          clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Abort aborted.", jobId));
+          abort = false;
+        } else {
+          sj = slaveJobs[jobId];
+        }
+      }
+
+      if (abort && !sj.Finished) {
+        sj.KillAppDomain();
+      }
+    }
+
+    private void HandleStopJob(object jobIdObj) {
+      Guid jobId = (Guid)jobIdObj;
+      bool stop = true;
+      SlaveJob sj = null;
+
+      lock (slaveJobs) {
+        if (!slaveJobs.ContainsKey(jobId)) {
+          clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Stop aborted.", jobId));
+          stop = false;
+        } else {
+          sj = slaveJobs[jobId];
+        }
+      }
+
+      if (stop && !sj.Finished) {
+        sj.StopJob();
+      }
+    }
+
+    private void HandlePauseJob(object jobIdObj) {
+      Guid jobId = (Guid)jobIdObj;
+      bool pause = true;
+      SlaveJob sj = null;
+
+      lock (slaveJobs) {
+        if (!slaveJobs.ContainsKey(jobId)) {
+          clientCom.LogMessage(string.Format("Job with id {0} doesn't exist. Pause aborted.", jobId));
+          pause = false;
+        } else {
+          sj = slaveJobs[jobId];
+        }
+      }
+
+      if (pause && !sj.Finished) {
+        sj.PauseJob();
       }
     }
@@ -372 +380 @@
     }
 
-    /// <summary>
-    /// Enqueues messages from the executor to the message queue. 
-    /// This is necessary if the core thread has to execute certain actions, e.g.
-    /// killing of an app domain. 
-    /// </summary>    
-    /// <returns>true if the calling method can continue execution, else false</returns>
-    public void EnqueueExecutorMessage<T>(Action<T> action, T parameter) {
-      ExecutorMessageContainer<T> container = new ExecutorMessageContainer<T>();
-      container.Callback = action;
-      container.CallbackParameter = parameter;
-      MessageQueue.GetInstance().AddMessage(container);
-    }
-
     public void RemoveSlaveJobFromList(Guid jobId) {
       lock (slaveJobs) {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveJob.cs

@@ -34 +34 @@
     }
 
-    public void CalculateJob(Guid jobId) {
+    public void PrepareJob(Guid jobId) {
+      JobId = jobId;
       Job job = wcfService.GetJob(jobId);
       if (job == null) throw new JobNotFoundException(jobId);
       coresNeeded = job.CoresNeeded;
       SlaveStatusInfo.IncrementUsedCores(coresNeeded);
+    }
+
+    public void CalculateJob() {
+      Job job = wcfService.GetJob(JobId);
+      if (job == null) throw new JobNotFoundException(JobId);
 
       JobData jobData = wcfService.GetJobData(job.Id);
-      if (jobData == null) throw new JobDataNotFoundException(jobId);
+      if (jobData == null) throw new JobDataNotFoundException(JobId);
       SlaveStatusInfo.IncrementJobsFetched();
       job = wcfService.UpdateJobState(job.Id, JobState.Calculating, null);
-      if (job == null) throw new JobNotFoundException(jobId);
+      if (job == null) throw new JobNotFoundException(JobId);
       StartJobInAppDomain(job, jobData);
     }
@@ -66 +72 @@
             SlaveStatusInfo.IncrementJobsAborted();
 
-            clientCom.LogMessage("Sending the stopped job with id: " + job.Id);
+            clientCom.LogMessage(string.Format("Sending the stopped job with id: {0}", job.Id));
             wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Aborted);
           }
           catch (Exception e) {
-            clientCom.LogMessage("Transmitting to server failed. Storing the paused job with id: " + job.Id + " to hdd (" + e.ToString() + ")");
+            clientCom.LogMessage(string.Format("Transmitting the stopped job with id {0} to server failed. Exception is: {1}", job.Id, e.ToString()));
           }
           finally {
@@ -101 +107 @@
           }
           catch (Exception e) {
-            clientCom.LogMessage("Transmitting to server failed. Storing the paused job with id: " + job.Id + " to hdd (" + e.ToString() + ")");
+            clientCom.LogMessage(string.Format("Transmitting the paused job with id {0} to server failed. Exception is: {1}", job.Id, e.ToString()));
           }
           finally {
@@ -138 +144 @@
     public void SendFinishedJob() {
       try {
-        clientCom.LogMessage("Getting the finished job with id: " + JobId);
+        clientCom.LogMessage(string.Format("Getting the finished job with id: {0} ", JobId));
         if (executor == null) {
           clientCom.LogMessage(string.Format("SendFinishedJob: Can't pause job with id {0} with uninitialized executor", JobId));
@@ -159 +165 @@
         try {
           JobData sJob = executor.GetFinishedJob();
-          clientCom.LogMessage("Sending the finished job with id: " + JobId);
+          clientCom.LogMessage(string.Format("Sending the finished job with id: {0}", JobId));
           wcfService.UpdateJobData(job, sJob, ConfigManager.Instance.GetClientInfo().Id, JobState.Finished);
         }
         catch (Exception e) {
-          clientCom.LogMessage("Transmitting to server failed. Storing the finished job with id: " + JobId + " to hdd (" + e.ToString() + ")");
+          clientCom.LogMessage(string.Format("Transmitting the job with id {0} to server failed. Exception is: {1}", job.Id, e.ToString()));
         }
         finally {
@@ -171 +177 @@
       }
       catch (Exception e) {
-        clientCom.LogMessage("SendFinishedJob: The following exception has been thrown: " + e.ToString());
+        clientCom.LogMessage(string.Format("SendFinishedJob: The following exception has been thrown: {0}", e.ToString()));
       }
     }
@@ -181 +187 @@
       JobId = job.Id;
 
-      clientCom.LogMessage("Received new job with id " + job.Id);
+      clientCom.LogMessage(string.Format("Received new job with id {0}", job.Id));
       clientCom.StatusChanged(ConfigManager.Instance.GetStatusForClientConsole());
 
@@ -190 +196 @@
       try {
         PluginCache.Instance.PreparePlugins(job, out configFileName);
-        clientCom.LogMessage("Plugins fetched for job " + job.Id);
+        clientCom.LogMessage(string.Format("Plugins fetched for job {0}", job.Id));
         pluginsPrepared = true;
       }
@@ -214 +220 @@
           executor.CoresNeeded = job.CoresNeeded;
           executor.MemoryNeeded = job.MemoryNeeded;
-          clientCom.LogMessage("Starting Executor for job " + job.Id);
+          clientCom.LogMessage(string.Format("Starting Executor for job {0}", job.Id));
 
           executor.Start(jobData.Data);
@@ -222 +228 @@
         }
         catch (Exception exception) {
-          clientCom.LogMessage("Creating the Appdomain and loading the job failed for job " + job.Id);
-          clientCom.LogMessage("Error thrown is: " + exception.ToString());
+          clientCom.LogMessage(string.Format("Creating the Appdomain and loading the job failed for job {0}", job.Id));
+          clientCom.LogMessage(string.Format("Error thrown is: {0}", exception.ToString()));
 
           if (executor != null && executor.CurrentException != string.Empty) {
@@ -243 +249 @@
     /// <param name="JobId">the GUID of the job</param>    
     public void KillAppDomain() {
-      clientCom.LogMessage("Shutting down Appdomain for Job " + JobId);
+      clientCom.LogMessage(string.Format("Shutting down Appdomain for Job {0}", JobId));
 
       try {
@@ -280 +286 @@
       }
       catch (Exception ex) {
-        clientCom.LogMessage("Exception when unloading the appdomain: " + ex.ToString());
+        clientCom.LogMessage(string.Format("Exception when unloading the appdomain: {0}", ex.ToString()));
       }
       finally {
@@ -291 +297 @@
 
     private void AppDomain_UnhandledException(object sender, UnhandledExceptionEventArgs e) {
-      clientCom.LogMessage("Exception in AppDomain: " + e.ExceptionObject.ToString());
+      clientCom.LogMessage(string.Format("Exception in AppDomain: ", e.ExceptionObject.ToString()));
       KillAppDomain();
     }