Changeset 5181

Timestamp:

12/29/10 15:50:03 (14 years ago)

Author:

cneumuel

Message:

#1260

• refined job-downloads for limited resources
• fixed minor bugs

Location:

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive

Files:

• Build.cmd (added)
• HeuristicLab.Hive.ExperimentManager/3.3/HiveJob.cs (modified) (1 diff)
• HeuristicLab.Hive.ExperimentManager/3.3/HiveJobDownloader.cs (modified) (5 diffs)
• HeuristicLab.Hive.ExperimentManager/3.3/app.config (modified) (1 diff)
• HeuristicLab.Hive.Server.Console/3.3/app.config (modified) (1 diff)
• HeuristicLab.Hive.Slave.Communication/3.3/app.config (modified) (1 diff)
• HeuristicLab.Hive.Slave.Core/3.3/ConfigurationManager/ConfigManager.cs (modified) (1 diff)
• HeuristicLab.Hive.Slave.Core/3.3/HeartbeatManager.cs (modified) (1 diff)
• PreBuildEvent.cmd (modified) (1 diff)

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/HiveJob.cs

@@ -327 +327 @@
           }
         } else {
-          childIsFinishedOptimizerDownloaded = optimizerJob.Optimizer.ExecutionState == ExecutionState.Stopped;
+          childIsFinishedOptimizerDownloaded = optimizerJob.Optimizer.ExecutionState == ExecutionState.Stopped || optimizerJob.Optimizer.ExecutionState == ExecutionState.Prepared;
         }
       }

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/HiveJobDownloader.cs

@@ -14 +14 @@
     private IEnumerable<Guid> jobIds;
     private List<Task<HiveJob>> tasks;
+    private bool abort = false;
 
     public bool IsFinished {
@@ -25 +26 @@
     public int FinishedCount {
       get {
+        var faulted = tasks.Where(t => t.Status == TaskStatus.Faulted);
+        if (faulted.Count() > 0) {
+          abort = true;
+          throw faulted.First().Exception;
+        }
         return tasks.Count(t => t.Status == TaskStatus.RanToCompletion ||
                                 t.Status == TaskStatus.Faulted ||
@@ -34 +40 @@
       get {
         var results = new Dictionary<Guid, HiveJob>();
-        foreach(var t in tasks) {
-          results.Add(t.Result.JobDto.Id, t.Result);
+        foreach (var t in tasks) {
+          if (t.Status == TaskStatus.Faulted) {
+            throw t.Exception;
+          }
+          if(t.Result != null) 
+            results.Add(t.Result.JobDto.Id, t.Result);
         }
         return results;
@@ -45 +55 @@
     }
 
-    
     public void StartAsync() {
+      abort = false;
       tasks = new List<Task<HiveJob>>();
       foreach (Guid jobId in jobIds) {
@@ -54 +64 @@
       }
     }
-    // use semaphore to ensure only few tasks download concurrently (so not too many connections are open)
-    private Semaphore taskSemaphore = new Semaphore(2, 2);
+    // use semaphore to ensure only few concurrenct connections and few SerializedJob objects in memory
+    private Semaphore downloadSemaphore = new Semaphore(2, 2);
+    private Semaphore deserializeSemaphore = new Semaphore(2, 2);
     protected SerializedJob DownloadJob(object jobId) {
-      taskSemaphore.WaitOne(Timeout.Infinite);
-      using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
-        return service.Obj.GetLastSerializedResult((Guid)jobId).Obj;
+      downloadSemaphore.WaitOne();
+      deserializeSemaphore.WaitOne();
+      SerializedJob result;
+      try {
+        if (abort) return null;
+        using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
+          result = service.Obj.GetLastSerializedResult((Guid)jobId).Obj;
+        }
       }
+      finally {
+        downloadSemaphore.Release();
+      }
+      return result;
     }
 
     protected HiveJob DeserializeJob(SerializedJob serializedJob) {
-      HiveJob job = new HiveJob(serializedJob, false);
-      job.Job.Prepare(); // reduce memory consumption.
-      taskSemaphore.Release();
-      job.JobDto = serializedJob.JobInfo;
-      return job;
+      try {
+        if (abort || serializedJob == null) return null;
+        HiveJob job = new HiveJob(serializedJob, false);
+        serializedJob.SerializedJobData = null; // reduce memory consumption.
+        job.Job.Prepare(); // reduce memory consumption.
+        job.JobDto = serializedJob.JobInfo;
+        return job;
+      }
+      finally {
+        deserializeSemaphore.Release();
+      }
     }
   }

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/app.config

@@ -29 +29 @@
     </bindings>
     <client>
-      <endpoint address="net.tcp://christoph:9001/Hive-3.3/ClientService.svc" binding="netTcpBinding" bindingConfiguration="ClientTcpStreamedEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientTcpStreamedEndpoint">
+      <endpoint address="net.tcp://services.heuristiclab.com:8000/Hive-3.3/ClientService.svc" binding="netTcpBinding" bindingConfiguration="ClientTcpStreamedEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientTcpStreamedEndpoint">
         <identity>
-          <dns value="localhost"/>
+          <dns value="services.heuristiclab.com"/>
         </identity>
       </endpoint>
-      <endpoint address="http://christoph/Hive-3.3/ClientService.svc" binding="wsHttpBinding" bindingConfiguration="ClientHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientHttpEndpoint">
+      <endpoint address="http://services.heuristiclab.com/Hive-3.3/ClientService.svc" binding="wsHttpBinding" bindingConfiguration="ClientHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientHttpEndpoint">
         <identity>
-          <certificate encodedValue="AwAAAAEAAAAUAAAAozaKcSPdw1Cdd57hw9mr3eCjxTIgAAAAAQAAAPIBAAAwggHuMIIBW6ADAgECAhBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTEwMTAxMjA2NTYyNloXDTM5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDFM6bQUgvCnsnBO9TZnOrC6x1riE2pXUzU6H38GYot674WNfir82fMh9+ojWSY/CiMzExcNWFtC3i8BG7giLYVwSanuwQ8QyRFYq3g5UMqdlm4/aof/4QMmS+SuRDsa8F4lbvdS9zBVFiyyUNYzlzO/rY08DrkCgx4X2IQGOvixwIDAQABo0kwRzBFBgNVHQEEPjA8gBCjMn9uvzNhU2c//E3RSYmUoRYwFDESMBAGA1UEAxMJbG9jYWxob3N0ghBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQADgYEADIf8as2VBfZEpwb/GuVifvPTR0Ud+dTCwTjBdMymqjNR5NTSi408DifssSTBVGIi7xkWbQDlQQHOZmVqVFoqfNbGrtm9w1MSK24HAEMXr0WFBltvldGKCvtYnZqf377qU5gq7IBXcJZi87bZ2l1M3icr6RJo4fIbnZOa5M430aw="/>
+          <certificate encodedValue="AwAAAAEAAAAUAAAAwK1+2oAmcy/mI2P2QjyiJRh0y60gAAAAAQAAACoCAAAwggImMIIBj6ADAgECAhAIkseQ2EEhgU720qJA61gqMA0GCSqGSIb3DQEBBAUAMCQxIjAgBgNVBAMTGXNlcnZpY2VzLmhldXJpc3RpY2xhYi5jb20wHhcNMTAwNTExMTExNDAyWhcNMzkxMjMxMjM1OTU5WjAkMSIwIAYDVQQDExlzZXJ2aWNlcy5oZXVyaXN0aWNsYWIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCq26Bwmwc7k+4W30qLQ2j+FInEL5BuH6opDY6CSlrtt3xQS/anrhvpbf3QghLDVINzcHkzbPmm/SguG4F85QLB6xO+tJaOvRo0iEK5g3c307vMIru7FJwk/OhplEQ5J1hbDgL3zOJlrWlgtqRVxCtVdF3XroI9BctOt1NkeKv9ewIDAQABo1kwVzBVBgNVHQEETjBMgBCjbgdYd4j5JgUuJ1Wo/GxroSYwJDEiMCAGA1UEAxMZc2VydmljZXMuaGV1cmlzdGljbGFiLmNvbYIQCJLHkNhBIYFO9tKiQOtYKjANBgkqhkiG9w0BAQQFAAOBgQAb/2xk2uQad68shSPl/uixWgvFI8WkxOTBopOLaLtDxwCeZ3mWVHdV9VnixHtThubnEBXAhYOCQSIXWtQuXFWO+gH3YyjTRJY5kTmXyuvBRTn3/so5SrQ7Rdlm9hf6E5YVX3tCjAy7ybUyaDUkQfmH5vmvgvpMzRfsJ1qhnUpJiQ=="/>
         </identity>
       </endpoint>
     </client>
-
+    
   </system.serviceModel>
 <startup><supportedRuntime version="v2.0.50727"/></startup></configuration>

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.Server.Console/3.3/app.config

@@ -26 +26 @@
     </bindings>
     <client>
-      <endpoint address="http://christoph/Hive-3.3/ServerConsoleService.svc" behaviorConfiguration="ServerConsoleBehaviour" binding="wsHttpBinding" bindingConfiguration="ServerConsoleHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IServerConsoleFacade" name="ServerConsoleHttpEndpoint">
+      <endpoint address="http://services.heuristiclab.com/Hive-3.3/ServerConsoleService.svc" behaviorConfiguration="ServerConsoleBehaviour" binding="wsHttpBinding" bindingConfiguration="ServerConsoleHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IServerConsoleFacade" name="ServerConsoleHttpEndpoint">
         <identity>
-          <certificate encodedValue="AwAAAAEAAAAUAAAAozaKcSPdw1Cdd57hw9mr3eCjxTIgAAAAAQAAAPIBAAAwggHuMIIBW6ADAgECAhBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTEwMTAxMjA2NTYyNloXDTM5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDFM6bQUgvCnsnBO9TZnOrC6x1riE2pXUzU6H38GYot674WNfir82fMh9+ojWSY/CiMzExcNWFtC3i8BG7giLYVwSanuwQ8QyRFYq3g5UMqdlm4/aof/4QMmS+SuRDsa8F4lbvdS9zBVFiyyUNYzlzO/rY08DrkCgx4X2IQGOvixwIDAQABo0kwRzBFBgNVHQEEPjA8gBCjMn9uvzNhU2c//E3RSYmUoRYwFDESMBAGA1UEAxMJbG9jYWxob3N0ghBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQADgYEADIf8as2VBfZEpwb/GuVifvPTR0Ud+dTCwTjBdMymqjNR5NTSi408DifssSTBVGIi7xkWbQDlQQHOZmVqVFoqfNbGrtm9w1MSK24HAEMXr0WFBltvldGKCvtYnZqf377qU5gq7IBXcJZi87bZ2l1M3icr6RJo4fIbnZOa5M430aw="/>
+          <certificate encodedValue="AwAAAAEAAAAUAAAAwK1+2oAmcy/mI2P2QjyiJRh0y60gAAAAAQAAACoCAAAwggImMIIBj6ADAgECAhAIkseQ2EEhgU720qJA61gqMA0GCSqGSIb3DQEBBAUAMCQxIjAgBgNVBAMTGXNlcnZpY2VzLmhldXJpc3RpY2xhYi5jb20wHhcNMTAwNTExMTExNDAyWhcNMzkxMjMxMjM1OTU5WjAkMSIwIAYDVQQDExlzZXJ2aWNlcy5oZXVyaXN0aWNsYWIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCq26Bwmwc7k+4W30qLQ2j+FInEL5BuH6opDY6CSlrtt3xQS/anrhvpbf3QghLDVINzcHkzbPmm/SguG4F85QLB6xO+tJaOvRo0iEK5g3c307vMIru7FJwk/OhplEQ5J1hbDgL3zOJlrWlgtqRVxCtVdF3XroI9BctOt1NkeKv9ewIDAQABo1kwVzBVBgNVHQEETjBMgBCjbgdYd4j5JgUuJ1Wo/GxroSYwJDEiMCAGA1UEAxMZc2VydmljZXMuaGV1cmlzdGljbGFiLmNvbYIQCJLHkNhBIYFO9tKiQOtYKjANBgkqhkiG9w0BAQQFAAOBgQAb/2xk2uQad68shSPl/uixWgvFI8WkxOTBopOLaLtDxwCeZ3mWVHdV9VnixHtThubnEBXAhYOCQSIXWtQuXFWO+gH3YyjTRJY5kTmXyuvBRTn3/so5SrQ7Rdlm9hf6E5YVX3tCjAy7ybUyaDUkQfmH5vmvgvpMzRfsJ1qhnUpJiQ=="/>
         </identity>
       </endpoint>

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.Slave.Communication/3.3/app.config

@@ -27 +27 @@
     </bindings>
     <client>
-      <endpoint address="http://christoph/Hive-3.3/SlaveService.svc" binding="wsHttpBinding" bindingConfiguration="SlaveHttpEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveHttpEndpoint">
+      <endpoint address="http://services.heuristiclab.com/Hive-3.3/SlaveService.svc" binding="wsHttpBinding" bindingConfiguration="SlaveHttpEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveHttpEndpoint">
         <identity>
-          <certificate encodedValue="AwAAAAEAAAAUAAAAozaKcSPdw1Cdd57hw9mr3eCjxTIgAAAAAQAAAPIBAAAwggHuMIIBW6ADAgECAhBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTEwMTAxMjA2NTYyNloXDTM5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDFM6bQUgvCnsnBO9TZnOrC6x1riE2pXUzU6H38GYot674WNfir82fMh9+ojWSY/CiMzExcNWFtC3i8BG7giLYVwSanuwQ8QyRFYq3g5UMqdlm4/aof/4QMmS+SuRDsa8F4lbvdS9zBVFiyyUNYzlzO/rY08DrkCgx4X2IQGOvixwIDAQABo0kwRzBFBgNVHQEEPjA8gBCjMn9uvzNhU2c//E3RSYmUoRYwFDESMBAGA1UEAxMJbG9jYWxob3N0ghBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQADgYEADIf8as2VBfZEpwb/GuVifvPTR0Ud+dTCwTjBdMymqjNR5NTSi408DifssSTBVGIi7xkWbQDlQQHOZmVqVFoqfNbGrtm9w1MSK24HAEMXr0WFBltvldGKCvtYnZqf377qU5gq7IBXcJZi87bZ2l1M3icr6RJo4fIbnZOa5M430aw="/>
+          <certificate encodedValue="AwAAAAEAAAAUAAAAwK1+2oAmcy/mI2P2QjyiJRh0y60gAAAAAQAAACoCAAAwggImMIIBj6ADAgECAhAIkseQ2EEhgU720qJA61gqMA0GCSqGSIb3DQEBBAUAMCQxIjAgBgNVBAMTGXNlcnZpY2VzLmhldXJpc3RpY2xhYi5jb20wHhcNMTAwNTExMTExNDAyWhcNMzkxMjMxMjM1OTU5WjAkMSIwIAYDVQQDExlzZXJ2aWNlcy5oZXVyaXN0aWNsYWIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCq26Bwmwc7k+4W30qLQ2j+FInEL5BuH6opDY6CSlrtt3xQS/anrhvpbf3QghLDVINzcHkzbPmm/SguG4F85QLB6xO+tJaOvRo0iEK5g3c307vMIru7FJwk/OhplEQ5J1hbDgL3zOJlrWlgtqRVxCtVdF3XroI9BctOt1NkeKv9ewIDAQABo1kwVzBVBgNVHQEETjBMgBCjbgdYd4j5JgUuJ1Wo/GxroSYwJDEiMCAGA1UEAxMZc2VydmljZXMuaGV1cmlzdGljbGFiLmNvbYIQCJLHkNhBIYFO9tKiQOtYKjANBgkqhkiG9w0BAQQFAAOBgQAb/2xk2uQad68shSPl/uixWgvFI8WkxOTBopOLaLtDxwCeZ3mWVHdV9VnixHtThubnEBXAhYOCQSIXWtQuXFWO+gH3YyjTRJY5kTmXyuvBRTn3/so5SrQ7Rdlm9hf6E5YVX3tCjAy7ybUyaDUkQfmH5vmvgvpMzRfsJ1qhnUpJiQ=="/>
         </identity>
       </endpoint>
-      <endpoint address="net.tcp://christoph:9001/Hive-3.3/SlaveService.svc" binding="netTcpBinding" bindingConfiguration="SlaveTcpStreamedEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveTcpStreamedEndpoint">
+      <endpoint address="net.tcp://services.heuristiclab.com:8000/Hive-3.3/SlaveService.svc" binding="netTcpBinding" bindingConfiguration="SlaveTcpStreamedEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveTcpStreamedEndpoint">
         <identity>
-          <dns value="localhost"/>
+          <dns value="services.heuristiclab.com"/>
         </identity>
       </endpoint>

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.Slave.Core/3.3/ConfigurationManager/ConfigManager.cs

@@ -104 +104 @@
     }
 
-    public Dictionary<Guid, TimeSpan> GetExecutionTimeOfAllJobs() {
+    public Dictionary<Guid, TimeSpan> GetExecutionTimeOfRunningJobs() {
       Dictionary<Guid, TimeSpan> prog = new Dictionary<Guid, TimeSpan>();
       Dictionary<Guid, Executor> engines = Core.ExecutionEngines;
       lock (engines) {
         foreach (KeyValuePair<Guid, Executor> kvp in engines) {
-          Executor e = kvp.Value;
-          prog[e.JobId] = e.ExecutionTime;
+          if (kvp.Value.ExecutionState == HeuristicLab.Core.ExecutionState.Started) { // only report from started jobs. this ensures that hung jobs will get an abort after some timeout. also if a job is already prepared (after it stopped), the executionTime would be 00:00:00
+            Executor e = kvp.Value;
+            prog[e.JobId] = e.ExecutionTime;
+          }
         }
       }

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.Slave.Core/3.3/HeartbeatManager.cs

@@ -98 +98 @@
                 FreeCores = info.NrOfCores.HasValue ? info.NrOfCores.Value - ConfigManager.Instance.GetUsedCores() : 0,
                 FreeMemory = GetFreeMemory(),
-                JobProgress = ConfigManager.Instance.GetExecutionTimeOfAllJobs(),
+                JobProgress = ConfigManager.Instance.GetExecutionTimeOfRunningJobs(),
                 IsAllowedToCalculate = UptimeManager.Instance.IsAllowedToCalculate() && UptimeManager.Instance.CalendarAvailable
               };

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/PreBuildEvent.cmd

@@ -4 +4 @@
 REM SET configfile=f005pc.hagenberg.fhooe.at - app.config
 REM SET configfile=localhost - app.config
-REM SET configfile=services.heuristiclab.com - app.config
-SET configfile=christoph - app.config
+SET configfile=services.heuristiclab.com - app.config
+REM SET configfile=christoph - app.config
 
 IF EXIST "%configfile%" (