Changeset 5450

Timestamp:

02/07/11 11:44:15 (13 years ago)

Author:

ascheibe

Message:

#1233

• added Pause/Stop/Abort mechanisms to the slave
• added Pause to Jobs

Location:

branches/HeuristicLab.Hive-3.4/sources

Files:

• HeuristicLab.Clients.Hive.Slave.Tests/Mocks/MockJob.cs (modified) (1 diff)
• HeuristicLab.Clients.Hive.Slave.Views/3.4/SlaveItem.cs (modified) (2 diffs)
• HeuristicLab.Clients.Hive.Slave/3.4/Core.cs (modified) (10 diffs)
• HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs (modified) (4 diffs)
• HeuristicLab.Clients.Hive.Slave/3.4/ServiceContracts/ISlaveCommunication.cs (modified) (1 diff)
• HeuristicLab.Clients.Hive.Slave/3.4/SlaveClientCom.cs (modified) (2 diffs)
• HeuristicLab.Clients.Hive.Slave/3.4/SlaveCommunicationService.cs (modified) (2 diffs)
• HeuristicLab.Clients.Hive/3.4/Jobs/OptimizerJob.cs (modified) (4 diffs)
• HeuristicLab.Hive/3.4/IJob.cs (modified) (5 diffs)
• HeuristicLab.Services.Hive.Common/3.4/MessageContainer.cs (modified) (1 diff)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Tests/Mocks/MockJob.cs

@@ -204 +204 @@
     }
 
+    public event EventHandler JobPaused;
+    protected virtual void OnJobPaused() {
+      EventHandler handler = JobPaused;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
     #endregion
   }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Views/3.4/SlaveItem.cs

@@ -95 +95 @@
       try {
         if (pipeFactory.State != CommunicationState.Faulted && pipeFactory.State != CommunicationState.Closed)
-          pipeProxy.SoftPause();
+          pipeProxy.StopAll();
       }
       catch (Exception e) {
@@ -105 +105 @@
       try {
         if (pipeFactory.State != CommunicationState.Faulted && pipeFactory.State != CommunicationState.Closed)
-          pipeProxy.HardPause();
+          pipeProxy.PauseAll();
       }
       catch (Exception e) {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs

@@ -40 +40 @@
 
     //TODO: this class should be a singleton; there is only one instance, the reference is meanwhile save in TheCore
-    public static Core theCore;
+    public static Core TheCore;
 
     public static bool abortRequested { get; set; }
@@ -66 +66 @@
 
     public Core() {
-      theCore = this;
+      TheCore = this;
     }
 
@@ -140 +140 @@
       } else if (container is MessageContainer) {
         switch (container.Message) {
-          //Server requests to abort a job
-          case MessageContainer.MessageType.AbortJob:
-            if (engines.ContainsKey(container.JobId))
-              try {
-                engines[container.JobId].Abort();
-              }
-              catch (AppDomainUnloadedException) {
-                // appdomain already unloaded. Finishing job probably ongoing
-              } else
-              ClientCom.LogMessage("AbortJob: Engine doesn't exist");
-            break;
-
           //Pull a Job from the Server
           case MessageContainer.MessageType.CalculateJob:
@@ -160 +148 @@
             break;
 
-          //Hard shutdown of the client
           case MessageContainer.MessageType.ShutdownSlave:
             ShutdownCore();
             break;
-          case MessageContainer.MessageType.HardPause:
-            doHardPause();
-            break;
-          case MessageContainer.MessageType.SoftPause:
-            doSoftPause();
+          case MessageContainer.MessageType.StopAll:
+            DoStopAll();
+            break;
+          case MessageContainer.MessageType.PauseAll:
+            DoPauseAll();
+            break;
+          case MessageContainer.MessageType.AbortAll:
+            DoAbortAll();
+            break;
+          case MessageContainer.MessageType.AbortJob:
+            KillAppDomain(container.JobId);
+            break;
+          case MessageContainer.MessageType.StopJob:
+            DoStopJob(container.JobId);
+            break;
+          case MessageContainer.MessageType.PauseJob:
+            DoPauseJob(container.JobId);
             break;
           case MessageContainer.MessageType.Restart:
-            doRestart();
+            DoStartSlave();
             break;
         }
@@ -179 +178 @@
     }
 
-    /// <summary>
-    /// reinitializes everything and continues operation, 
-    /// can be called after SoftPause() or HardPause()
-    /// </summary>
-    public void Restart() {
-      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.Restart);
-      MessageQueue.GetInstance().AddMessage(mc);
-    }
-
-    private void doRestart() {
-      ClientCom.LogMessage("Restart received");
-      StartHeartbeats();
-      ClientCom.LogMessage("Restart done");
-    }
-
-    /// <summary>
-    /// wait for jobs to finish, then pause client
-    /// </summary>
-    public void SoftPause() {
-      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.SoftPause);
-      MessageQueue.GetInstance().AddMessage(mc);
-    }
-
-    private void doSoftPause() {
-      ClientCom.LogMessage("Soft pause received");
-
-      //TODO: jobs get removed from Jobs map, is this a problem?
-      foreach (Job job in Jobs.Values) {
+    private void DoPauseJob(Guid guid) {
+      Job job = Jobs[guid];
+
+      if (job != null) {
         engines[job.Id].Pause();
         JobData sJob = engines[job.Id].GetFinishedJob();
@@ -224 +199 @@
         }
       }
-
+    }
+
+    private void DoStopJob(Guid guid) {
+      Job job = Jobs[guid];
+
+      if (job != null) {
+        engines[job.Id].Stop();
+        JobData sJob = engines[job.Id].GetFinishedJob();
+        job.Exception = engines[job.Id].CurrentException;
+        job.ExecutionTime = engines[job.Id].ExecutionTime;
+
+        try {
+          ClientCom.LogMessage("Sending the stoppped job with id: " + job.Id);
+          wcfService.UpdateJob(job, sJob);
+          SlaveStatusInfo.JobsProcessed++;    //TODO: count or not count, thats the question
+        }
+        catch (Exception e) {
+          ClientCom.LogMessage("Transmitting to server failed. Storing the paused job with id: " + job.Id + " to hdd (" + e.ToString() + ")");
+        }
+        finally {
+          KillAppDomain(job.Id); // kill app-domain in every case          
+        }
+      }
+    }
+
+    /// <summary>
+    /// aborts all running jobs, no results are sent back
+    /// </summary>
+    private void DoAbortAll() {
+      List<Guid> guids = new List<Guid>();
+      foreach (Guid job in Jobs.Keys) {
+        guids.Add(job);
+      }
+
+      foreach (Guid g in guids) {
+        KillAppDomain(g);
+      }
+
+      ClientCom.LogMessage("Aborted all jobs!");
+    }
+
+    /// <summary>
+    /// wait for jobs to finish, then pause client
+    /// </summary>
+    private void DoPauseAll() {
+      ClientCom.LogMessage("Pause all received");
+
+      //copy guids because there will be removed items from 'Jobs'
+      List<Guid> guids = new List<Guid>();
+      foreach (Guid job in Jobs.Keys) {
+        guids.Add(job);
+      }
+
+      foreach (Guid g in guids) {
+        DoPauseJob(g);
+      }
+    }
+
+    /// <summary>
+    /// pause slave immediately
+    /// </summary>
+    private void DoStopAll() {
+      ClientCom.LogMessage("Stop all received");
+
+      //copy guids because there will be removed items from 'Jobs'
+      List<Guid> guids = new List<Guid>();
+      foreach (Guid job in Jobs.Keys) {
+        guids.Add(job);
+      }
+
+      foreach (Guid g in guids) {
+        DoStopJob(g);
+      }
+    }
+
+    /// <summary>
+    /// completly shudown slave
+    /// </summary>
+    public void Shutdown() {
+      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.ShutdownSlave);
+      MessageQueue.GetInstance().AddMessage(mc);
+      waitShutdownSem.WaitOne();
+    }
+
+    /// <summary>
+    /// complete shutdown, should be called before the the application is exited
+    /// </summary>
+    private void ShutdownCore() {
+      ClientCom.LogMessage("Shutdown Signal received");
+      ClientCom.LogMessage("Stopping heartbeat");
       heartbeatManager.StopHeartBeat();
-      WcfService.Instance.Disconnect();
-      ClientCom.LogMessage("Soft pause done");
-    }
-
-    /// <summary>
-    /// pause slave immediately
-    /// </summary>
-    public void HardPause() {
-      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.HardPause);
-      MessageQueue.GetInstance().AddMessage(mc);
-    }
-
-    private void doHardPause() {
-      ClientCom.LogMessage("Hard pause received");
-      heartbeatManager.StopHeartBeat();
+      abortRequested = true;
+      ClientCom.LogMessage("Logging out");
+
 
       lock (engines) {
@@ -251 +303 @@
       }
       WcfService.Instance.Disconnect();
-      ClientCom.LogMessage("Hard pause done");
-    }
-
-    public void Shutdown() {
-      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.ShutdownSlave);
-      MessageQueue.GetInstance().AddMessage(mc);
-      waitShutdownSem.WaitOne();
-    }
-
-    /// <summary>
-    /// hard shutdown, should be called before the the application is exited
-    /// </summary>
-    private void ShutdownCore() {
-      ClientCom.LogMessage("Shutdown Signal received");
-      ClientCom.LogMessage("Stopping heartbeat");
-      heartbeatManager.StopHeartBeat();
-      abortRequested = true;
-      ClientCom.LogMessage("Logging out");
-
-
-      lock (engines) {
-        ClientCom.LogMessage("engines locked");
-        foreach (KeyValuePair<Guid, AppDomain> kvp in appDomains) {
-          ClientCom.LogMessage("Shutting down Appdomain for " + kvp.Key);
-          appDomains[kvp.Key].UnhandledException -= new UnhandledExceptionEventHandler(appDomain_UnhandledException);
-          AppDomain.Unload(kvp.Value);
-        }
-      }
-      WcfService.Instance.Disconnect();
       ClientCom.Shutdown();
       SlaveClientCom.Close();
@@ -288 +311 @@
 
     /// <summary>
+    /// reinitializes everything and continues operation, 
+    /// can be called after Sleep()
+    /// </summary>  
+    private void DoStartSlave() {
+      ClientCom.LogMessage("Restart received");
+      StartHeartbeats();
+      ClientCom.LogMessage("Restart done");
+    }
+
+    /// <summary>
+    /// stop slave, except for client gui communication, 
+    /// primarily used by gui if core is running as windows service
+    /// </summary>
+    //TODO: do we need an AbortSleep?
+    private void Sleep() {
+      ClientCom.LogMessage("Sleep received");
+      heartbeatManager.StopHeartBeat();
+      DoStopAll();
+      WcfService.Instance.Disconnect();
+      ClientCom.LogMessage("Sleep done");
+    }
+
+    /// <summary>
     /// Pauses a job, which means sending it to the server and killing it locally;
     /// atm only used when executor is waiting for child jobs
@@ -293 +339 @@
     /// <param name="data"></param>
     [MethodImpl(MethodImplOptions.Synchronized)]
-    public void PauseJob(JobData data) {
+    public void PauseWaitJob(JobData data) {
       if (!Jobs.ContainsKey(data.JobId)) {
         ClientCom.LogMessage("Can't find job with id " + data.JobId);
@@ -377 +423 @@
               ClientCom.LogMessage("Created AppDomain");
               engine.JobId = myJob.Id;
-              engine.core = this;
+              engine.Core = this;
               ClientCom.LogMessage("Starting Engine for job " + myJob.Id);
               engines.Add(myJob.Id, engine);

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs

@@ -35 +35 @@
     public IJob Job { get; set; }
     private bool wasJobAborted = false;
-    public Core core { get; set; }
+    public Core Core { get; set; }
 
     private Exception currentException;
@@ -84 +84 @@
     }
 
-    public void StartOnlyJob() {
-      try {
-        Job.Start();
-      }
-      catch (Exception e) {
-        this.currentException = e;
-      }
+    public void Pause() {
+      Job.Pause();
+      
     }
 
-    public void Pause() {
-      Job.Pause();
-    }
-
-    public void Abort() {
+    public void Stop() {
       wasJobAborted = true;
       if ((ExecutionState == ExecutionState.Started) || (ExecutionState == ExecutionState.Paused)) {
@@ -153 +145 @@
       jdata.JobId = this.JobId;
 
-      core.PauseJob(jdata);
+      Core.PauseWaitJob(jdata);
     }
 
@@ -164 +156 @@
       HeuristicLab.Common.EventArgs<Exception> ex = (HeuristicLab.Common.EventArgs<Exception>)e;
       currentException = ex.Value;
-      core.SendFinishedJob(JobId);
+      Core.SendFinishedJob(JobId);
     }
 
     private void Job_JobStopped(object sender, EventArgs e) {
       if (wasJobAborted) {
-        core.KillAppDomain(JobId);
+        Core.KillAppDomain(JobId);
       } else {
-        core.SendFinishedJob(JobId);
+        Core.SendFinishedJob(JobId);
       }
     }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ServiceContracts/ISlaveCommunication.cs

@@ -34 +34 @@
 
 
-    //commands from gui to slave core
     [OperationContract]
     void Restart();
 
     [OperationContract]
-    void HardPause();
+    void PauseAll();
 
     [OperationContract]
-    void SoftPause();
+    void StopAll();
+
+    [OperationContract]
+    void AbortAll();
 
     [OperationContract]

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveClientCom.cs

@@ -21 +21 @@
 
 using System.ServiceModel;
-using HeuristicLab.Clients.Hive.Slave;
 using HeuristicLab.Clients.Hive.Slave.ServiceContracts;
 
@@ -48 +47 @@
 
     private SlaveClientCom() {
-      setupClientCom();
+      SetupClientCom();
     }
 
-    private void setupClientCom() {
+    private void SetupClientCom() {
       DummyListener dummy = new DummyListener();
       pipeFactory = new DuplexChannelFactory<ISlaveCommunication>(

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/SlaveCommunicationService.cs

@@ -23 +23 @@
 using System.ServiceModel;
 using HeuristicLab.Clients.Hive.Slave.ServiceContracts;
+using HeuristicLab.Services.Hive.Common;
 
 namespace HeuristicLab.Clients.Hive.Slave {
@@ -84 +85 @@
 
     public void Restart() {
-      Core.theCore.Restart();
-    }
-    public void HardPause() {
-      Core.theCore.HardPause();
+      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.Restart);
+      MessageQueue.GetInstance().AddMessage(mc);
     }
 
-    public void SoftPause() {
-      Core.theCore.SoftPause();
+    public void PauseAll() {
+      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.PauseAll);
+      MessageQueue.GetInstance().AddMessage(mc);
+    }
+
+    public void StopAll() {
+      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.StopAll);
+      MessageQueue.GetInstance().AddMessage(mc);
     }
 
     public void ShutdownSlave() {
-      Core.theCore.Shutdown();
+      Core.TheCore.Shutdown();
+    }
+
+    public void AbortAll() {
+      MessageContainer mc = new MessageContainer(MessageContainer.MessageType.AbortAll);
+      MessageQueue.GetInstance().AddMessage(mc);
     }
   }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/Jobs/OptimizerJob.cs

@@ -155 +155 @@
 
     public void Pause() {
-      throw new NotImplementedException();
+      optimizer.Pause();
     }
 
@@ -169 +169 @@
     protected virtual void OnJobStopped() {
       EventHandler handler = JobStopped;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
+
+    public event EventHandler JobPaused;
+    protected void OnJobPaused(object sender, EventArgs e) {
+      EventHandler handler = JobPaused;
       if (handler != null) handler(this, EventArgs.Empty);
     }
@@ -212 +218 @@
     protected virtual void RegisterEvents() {
       optimizer.Stopped += new EventHandler(optimizer_Stopped);
+      optimizer.Paused += new EventHandler(OnJobPaused);
       optimizer.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(optimizer_ExceptionOccurred);
       optimizer.DescriptionChanged += new EventHandler(optimizer_DescriptionChanged);
@@ -219 +226 @@
       optimizer.ToStringChanged += new EventHandler(optimizer_ToStringChanged);
     }
+
     protected virtual void DeregisterEvents() {
       optimizer.Stopped -= new EventHandler(optimizer_Stopped);
+      optimizer.Paused -= new EventHandler(OnJobPaused);
       optimizer.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(optimizer_ExceptionOccurred);
       optimizer.DescriptionChanged -= this.DescriptionChanged;

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Hive/3.4/IJob.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using HeuristicLab.Common;
 using HeuristicLab.Core;
-using HeuristicLab.Common;
 
 namespace HeuristicLab.Hive {
@@ -31 +31 @@
 
     ExecutionState ExecutionState { get; }
-    
+
     /// <summary>
     /// indicates wether it is possible to create childjobs from this job
@@ -47 +47 @@
     /// </summary>
     bool CollectChildJobs { get; set; }
-    
+
     void Prepare();
 
@@ -67 +67 @@
 
     event EventHandler JobStopped;
-        
+
+    event EventHandler JobPaused;
+
     /// <summary>
     /// When this event occurs the job wants to sleep until all his child jobs are finished
@@ -82 +84 @@
     /// </summary>
     event EventHandler DeleteChildJobs;
-    
+
 
   }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Common/3.4/MessageContainer.cs

@@ -37 +37 @@
       CalculateJob, // slave should calculate a job. the job is already assigned to the slave
       StopJob,   // slave should stop the job and submit results
+      StopAll,   // stop all and submit results
       AbortJob,  // slave should shut the job down immediately without submitting results
-      PauseJob,  // pause the job and submit the results
-      SoftPause,      // slave should stop all jobs, submit results and pause
-      HardPause,      // abort jobs and pause
+      AbortAll,  // slave should abort all jobs immediately
+      PauseJob,  // pause the job and submit the results   
+      PauseAll,  // pause all jobs and submit results
       Restart,        //restart operation after Soft/HardPause
       ShutdownSlave,  // slave should shutdown immediately without submitting results