Changeset 5227 for branches/HeuristicLab.Hive-3.3/sources

Timestamp:

01/06/11 13:17:56 (14 years ago)

Author:

cneumuel

Message:

#1260

• improved efficiency of HiveEngine

Location:

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive

Files:

• HeuristicLab.Hive.ExperimentManager/3.3/app.config (modified) (1 diff)
• HeuristicLab.Hive.Server.Console/3.3/app.config (modified) (1 diff)
• HeuristicLab.Hive.Slave.Communication/3.3/app.config (modified) (1 diff)
• HeuristicLab.HiveEngine/3.3/HiveEngine.cs (modified) (3 diffs)
• PreBuildEvent.cmd (modified) (1 diff)

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/app.config

@@ -29 +29 @@
     </bindings>
     <client>
-      <endpoint address="net.tcp://christoph:9001/Hive-3.3/ClientService.svc" binding="netTcpBinding" bindingConfiguration="ClientTcpStreamedEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientTcpStreamedEndpoint">
+      <endpoint address="net.tcp://services.heuristiclab.com:8000/Hive-3.3/ClientService.svc" binding="netTcpBinding" bindingConfiguration="ClientTcpStreamedEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientTcpStreamedEndpoint">
         <identity>
-          <dns value="localhost"/>
+          <dns value="services.heuristiclab.com"/>
         </identity>
       </endpoint>
-      <endpoint address="http://christoph/Hive-3.3/ClientService.svc" binding="wsHttpBinding" bindingConfiguration="ClientHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientHttpEndpoint">
+      <endpoint address="http://services.heuristiclab.com/Hive-3.3/ClientService.svc" binding="wsHttpBinding" bindingConfiguration="ClientHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IClientFacade" name="ClientHttpEndpoint">
         <identity>
-          <certificate encodedValue="AwAAAAEAAAAUAAAAozaKcSPdw1Cdd57hw9mr3eCjxTIgAAAAAQAAAPIBAAAwggHuMIIBW6ADAgECAhBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTEwMTAxMjA2NTYyNloXDTM5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDFM6bQUgvCnsnBO9TZnOrC6x1riE2pXUzU6H38GYot674WNfir82fMh9+ojWSY/CiMzExcNWFtC3i8BG7giLYVwSanuwQ8QyRFYq3g5UMqdlm4/aof/4QMmS+SuRDsa8F4lbvdS9zBVFiyyUNYzlzO/rY08DrkCgx4X2IQGOvixwIDAQABo0kwRzBFBgNVHQEEPjA8gBCjMn9uvzNhU2c//E3RSYmUoRYwFDESMBAGA1UEAxMJbG9jYWxob3N0ghBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQADgYEADIf8as2VBfZEpwb/GuVifvPTR0Ud+dTCwTjBdMymqjNR5NTSi408DifssSTBVGIi7xkWbQDlQQHOZmVqVFoqfNbGrtm9w1MSK24HAEMXr0WFBltvldGKCvtYnZqf377qU5gq7IBXcJZi87bZ2l1M3icr6RJo4fIbnZOa5M430aw="/>
+          <certificate encodedValue="AwAAAAEAAAAUAAAAwK1+2oAmcy/mI2P2QjyiJRh0y60gAAAAAQAAACoCAAAwggImMIIBj6ADAgECAhAIkseQ2EEhgU720qJA61gqMA0GCSqGSIb3DQEBBAUAMCQxIjAgBgNVBAMTGXNlcnZpY2VzLmhldXJpc3RpY2xhYi5jb20wHhcNMTAwNTExMTExNDAyWhcNMzkxMjMxMjM1OTU5WjAkMSIwIAYDVQQDExlzZXJ2aWNlcy5oZXVyaXN0aWNsYWIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCq26Bwmwc7k+4W30qLQ2j+FInEL5BuH6opDY6CSlrtt3xQS/anrhvpbf3QghLDVINzcHkzbPmm/SguG4F85QLB6xO+tJaOvRo0iEK5g3c307vMIru7FJwk/OhplEQ5J1hbDgL3zOJlrWlgtqRVxCtVdF3XroI9BctOt1NkeKv9ewIDAQABo1kwVzBVBgNVHQEETjBMgBCjbgdYd4j5JgUuJ1Wo/GxroSYwJDEiMCAGA1UEAxMZc2VydmljZXMuaGV1cmlzdGljbGFiLmNvbYIQCJLHkNhBIYFO9tKiQOtYKjANBgkqhkiG9w0BAQQFAAOBgQAb/2xk2uQad68shSPl/uixWgvFI8WkxOTBopOLaLtDxwCeZ3mWVHdV9VnixHtThubnEBXAhYOCQSIXWtQuXFWO+gH3YyjTRJY5kTmXyuvBRTn3/so5SrQ7Rdlm9hf6E5YVX3tCjAy7ybUyaDUkQfmH5vmvgvpMzRfsJ1qhnUpJiQ=="/>
         </identity>
       </endpoint>
     </client>
-
+    
   </system.serviceModel>
 <startup><supportedRuntime version="v2.0.50727"/></startup></configuration>

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.Server.Console/3.3/app.config

@@ -26 +26 @@
     </bindings>
     <client>
-      <endpoint address="http://christoph/Hive-3.3/ServerConsoleService.svc" behaviorConfiguration="ServerConsoleBehaviour" binding="wsHttpBinding" bindingConfiguration="ServerConsoleHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IServerConsoleFacade" name="ServerConsoleHttpEndpoint">
+      <endpoint address="http://services.heuristiclab.com/Hive-3.3/ServerConsoleService.svc" behaviorConfiguration="ServerConsoleBehaviour" binding="wsHttpBinding" bindingConfiguration="ServerConsoleHttpEndpoint" contract="HeuristicLab.Hive.Contracts.Interfaces.IServerConsoleFacade" name="ServerConsoleHttpEndpoint">
         <identity>
-          <certificate encodedValue="AwAAAAEAAAAUAAAAozaKcSPdw1Cdd57hw9mr3eCjxTIgAAAAAQAAAPIBAAAwggHuMIIBW6ADAgECAhBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTEwMTAxMjA2NTYyNloXDTM5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDFM6bQUgvCnsnBO9TZnOrC6x1riE2pXUzU6H38GYot674WNfir82fMh9+ojWSY/CiMzExcNWFtC3i8BG7giLYVwSanuwQ8QyRFYq3g5UMqdlm4/aof/4QMmS+SuRDsa8F4lbvdS9zBVFiyyUNYzlzO/rY08DrkCgx4X2IQGOvixwIDAQABo0kwRzBFBgNVHQEEPjA8gBCjMn9uvzNhU2c//E3RSYmUoRYwFDESMBAGA1UEAxMJbG9jYWxob3N0ghBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQADgYEADIf8as2VBfZEpwb/GuVifvPTR0Ud+dTCwTjBdMymqjNR5NTSi408DifssSTBVGIi7xkWbQDlQQHOZmVqVFoqfNbGrtm9w1MSK24HAEMXr0WFBltvldGKCvtYnZqf377qU5gq7IBXcJZi87bZ2l1M3icr6RJo4fIbnZOa5M430aw="/>
+          <certificate encodedValue="AwAAAAEAAAAUAAAAwK1+2oAmcy/mI2P2QjyiJRh0y60gAAAAAQAAACoCAAAwggImMIIBj6ADAgECAhAIkseQ2EEhgU720qJA61gqMA0GCSqGSIb3DQEBBAUAMCQxIjAgBgNVBAMTGXNlcnZpY2VzLmhldXJpc3RpY2xhYi5jb20wHhcNMTAwNTExMTExNDAyWhcNMzkxMjMxMjM1OTU5WjAkMSIwIAYDVQQDExlzZXJ2aWNlcy5oZXVyaXN0aWNsYWIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCq26Bwmwc7k+4W30qLQ2j+FInEL5BuH6opDY6CSlrtt3xQS/anrhvpbf3QghLDVINzcHkzbPmm/SguG4F85QLB6xO+tJaOvRo0iEK5g3c307vMIru7FJwk/OhplEQ5J1hbDgL3zOJlrWlgtqRVxCtVdF3XroI9BctOt1NkeKv9ewIDAQABo1kwVzBVBgNVHQEETjBMgBCjbgdYd4j5JgUuJ1Wo/GxroSYwJDEiMCAGA1UEAxMZc2VydmljZXMuaGV1cmlzdGljbGFiLmNvbYIQCJLHkNhBIYFO9tKiQOtYKjANBgkqhkiG9w0BAQQFAAOBgQAb/2xk2uQad68shSPl/uixWgvFI8WkxOTBopOLaLtDxwCeZ3mWVHdV9VnixHtThubnEBXAhYOCQSIXWtQuXFWO+gH3YyjTRJY5kTmXyuvBRTn3/so5SrQ7Rdlm9hf6E5YVX3tCjAy7ybUyaDUkQfmH5vmvgvpMzRfsJ1qhnUpJiQ=="/>
         </identity>
       </endpoint>

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.Slave.Communication/3.3/app.config

@@ -27 +27 @@
     </bindings>
     <client>
-      <endpoint address="http://christoph/Hive-3.3/SlaveService.svc" binding="wsHttpBinding" bindingConfiguration="SlaveHttpEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveHttpEndpoint">
+      <endpoint address="http://services.heuristiclab.com/Hive-3.3/SlaveService.svc" binding="wsHttpBinding" bindingConfiguration="SlaveHttpEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveHttpEndpoint">
         <identity>
-          <certificate encodedValue="AwAAAAEAAAAUAAAAozaKcSPdw1Cdd57hw9mr3eCjxTIgAAAAAQAAAPIBAAAwggHuMIIBW6ADAgECAhBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MB4XDTEwMTAxMjA2NTYyNloXDTM5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDFM6bQUgvCnsnBO9TZnOrC6x1riE2pXUzU6H38GYot674WNfir82fMh9+ojWSY/CiMzExcNWFtC3i8BG7giLYVwSanuwQ8QyRFYq3g5UMqdlm4/aof/4QMmS+SuRDsa8F4lbvdS9zBVFiyyUNYzlzO/rY08DrkCgx4X2IQGOvixwIDAQABo0kwRzBFBgNVHQEEPjA8gBCjMn9uvzNhU2c//E3RSYmUoRYwFDESMBAGA1UEAxMJbG9jYWxob3N0ghBhrVLVbjrtvUe4zWuGbVeuMAkGBSsOAwIdBQADgYEADIf8as2VBfZEpwb/GuVifvPTR0Ud+dTCwTjBdMymqjNR5NTSi408DifssSTBVGIi7xkWbQDlQQHOZmVqVFoqfNbGrtm9w1MSK24HAEMXr0WFBltvldGKCvtYnZqf377qU5gq7IBXcJZi87bZ2l1M3icr6RJo4fIbnZOa5M430aw="/>
+          <certificate encodedValue="AwAAAAEAAAAUAAAAwK1+2oAmcy/mI2P2QjyiJRh0y60gAAAAAQAAACoCAAAwggImMIIBj6ADAgECAhAIkseQ2EEhgU720qJA61gqMA0GCSqGSIb3DQEBBAUAMCQxIjAgBgNVBAMTGXNlcnZpY2VzLmhldXJpc3RpY2xhYi5jb20wHhcNMTAwNTExMTExNDAyWhcNMzkxMjMxMjM1OTU5WjAkMSIwIAYDVQQDExlzZXJ2aWNlcy5oZXVyaXN0aWNsYWIuY29tMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCq26Bwmwc7k+4W30qLQ2j+FInEL5BuH6opDY6CSlrtt3xQS/anrhvpbf3QghLDVINzcHkzbPmm/SguG4F85QLB6xO+tJaOvRo0iEK5g3c307vMIru7FJwk/OhplEQ5J1hbDgL3zOJlrWlgtqRVxCtVdF3XroI9BctOt1NkeKv9ewIDAQABo1kwVzBVBgNVHQEETjBMgBCjbgdYd4j5JgUuJ1Wo/GxroSYwJDEiMCAGA1UEAxMZc2VydmljZXMuaGV1cmlzdGljbGFiLmNvbYIQCJLHkNhBIYFO9tKiQOtYKjANBgkqhkiG9w0BAQQFAAOBgQAb/2xk2uQad68shSPl/uixWgvFI8WkxOTBopOLaLtDxwCeZ3mWVHdV9VnixHtThubnEBXAhYOCQSIXWtQuXFWO+gH3YyjTRJY5kTmXyuvBRTn3/so5SrQ7Rdlm9hf6E5YVX3tCjAy7ybUyaDUkQfmH5vmvgvpMzRfsJ1qhnUpJiQ=="/>
         </identity>
       </endpoint>
-      <endpoint address="net.tcp://christoph:9001/Hive-3.3/SlaveService.svc" binding="netTcpBinding" bindingConfiguration="SlaveTcpStreamedEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveTcpStreamedEndpoint">
+      <endpoint address="net.tcp://services.heuristiclab.com:8000/Hive-3.3/SlaveService.svc" binding="netTcpBinding" bindingConfiguration="SlaveTcpStreamedEndpoint" contract="SlaveFacade.ISlaveFacade" name="SlaveTcpStreamedEndpoint">
         <identity>
-          <dns value="localhost"/>
+          <dns value="services.heuristiclab.com"/>
         </identity>
       </endpoint>

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.HiveEngine/3.3/HiveEngine.cs

@@ -98 +98 @@
 
           if (operation.Operator.Breakpoint) {
-            Log.LogMessage(string.Format("Breakpoint: {0}", operation.Operator.Name != string.Empty ? operation.Operator.Name : operation.Operator.ItemName));
+            LogMessage(string.Format("Breakpoint: {0}", operation.Operator.Name != string.Empty ? operation.Operator.Name : operation.Operator.ItemName));
             Pause();
           }
@@ -136 +136 @@
     /// <param name="jobDict"></param>
     private void ExecuteOnHive(IDictionary<IOperation, OperationJob> jobDict) {
-      Log.LogMessage(string.Format("Executing {0} operations on the hive.", jobDict.Count));
+      LogMessage(string.Format("Executing {0} operations on the hive.", jobDict.Count));
+      Semaphore concurrentUploads = new Semaphore(6, 6);
       try {
         IDictionary<Guid, IOperation> jobIds = new Dictionary<Guid, IOperation>();
+        List<Guid> remainingJobIds = new List<Guid>();
+        IDictionary<Guid, int> jobNumbers = new Dictionary<Guid, int>(); // for better readability of log
         JobResultList results;
-
-        using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
-          List<JobDto> jobs = new List<JobDto>();
-          foreach (var kvp in jobDict) {
-            // shuffle random variable to avoid the same random sequence in each operation
-            IRandom random = FindRandomParameter(kvp.Key as IExecutionContext);
-            if (random != null)
-              random.Reset(random.Next());
-
+        var pluginsNeeded = ApplicationManager.Manager.Plugins.Select(x => new HivePluginInfoDto { Name = x.Name, Version = x.Version }).ToList();
+
+        List<JobDto> jobs = new List<JobDto>();
+        foreach (var kvp in jobDict) {
+
+          // shuffle random variable to avoid the same random sequence in each operation; todo: does not yet work (it cannot find the random variable)
+          IRandom random = FindRandomParameter(kvp.Key as IExecutionContext);
+          if (random != null)
+            random.Reset(random.Next());
+
+          Task.Factory.StartNew((operationJob) => {
             var groups = ResourceIds.Split(';');
             SerializedJob serializedJob = new SerializedJob();
-            serializedJob.SerializedJobData = SerializedJob.Serialize(kvp.Value);
+            serializedJob.SerializedJobData = SerializedJob.Serialize(operationJob);
             serializedJob.JobInfo = new JobDto();
             serializedJob.JobInfo.State = JobState.Offline;
             serializedJob.JobInfo.CoresNeeded = 1;
-            serializedJob.JobInfo.PluginsNeeded = ApplicationManager.Manager.Plugins.Select(x => new HivePluginInfoDto { Name = x.Name, Version = x.Version }).ToList();
-            ResponseObject<JobDto> response = service.Obj.AddJobWithGroupStrings(serializedJob, groups);
+            serializedJob.JobInfo.PluginsNeeded = pluginsNeeded;
+            ResponseObject<JobDto> response;
+            concurrentUploads.WaitOne();
+            using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
+              response = service.Obj.AddJobWithGroupStrings(serializedJob, groups);
+            }
+            concurrentUploads.Release();
             jobs.Add(response.Obj);
             jobIds.Add(response.Obj.Id, kvp.Key);
-          }
-          results = service.Obj.GetJobResults(jobIds.Keys).Obj;
-        }
-
-        while (!results.All(
-            x => x.State == JobState.Finished ||
-            x.State == JobState.Failed ||
-            x.State == JobState.Aborted)) {
+            lock (remainingJobIds) {
+              remainingJobIds.Add(response.Obj.Id);
+              jobNumbers.Add(response.Obj.Id, remainingJobIds.Count);
+            }
+            LogMessage(string.Format("Submitted job #{0} (id: {1})", jobNumbers[response.Obj.Id], response.Obj.Id));
+          }, kvp.Value);
+        }
+
+        while (remainingJobIds.Count != jobDict.Count) {
+          Thread.Sleep(1000);
+        }
+
+        LogMessage("Waiting for results...");
+        int jobsFinishedCount = 0;
+        Semaphore concurrentDownloads = new Semaphore(4, 4);
+        while (remainingJobIds.Count > 0) {
           Thread.Sleep(5000);
           using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
-            results = service.Obj.GetJobResults(jobIds.Keys).Obj;
-          }
-        }
-
-        // all finished
-        using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
-          foreach (Guid jobId in jobIds.Keys) {
-            SerializedJob serializedJob = service.Obj.GetLastSerializedResult(jobId).Obj;
-            OperationJob operationJob = SerializedJob.Deserialize<OperationJob>(serializedJob.SerializedJobData);
-            jobDict[jobIds[jobId]] = operationJob;
-          }
-        }
-
+            results = service.Obj.GetJobResults(remainingJobIds).Obj;
+          }
+          foreach (var result in results) {
+            if (result.State == JobState.Finished) {
+              Task.Factory.StartNew((jobIdObj) => {
+                Guid jobId = (Guid)jobIdObj;
+                SerializedJob serializedJob;
+                concurrentDownloads.WaitOne();
+                using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
+                  serializedJob = service.Obj.GetLastSerializedResult(jobId).Obj;
+                }
+                concurrentDownloads.Release();
+                OperationJob operationJob = SerializedJob.Deserialize<OperationJob>(serializedJob.SerializedJobData);
+                jobDict[jobIds[jobId]] = operationJob;
+                LogMessage(string.Format("Downloaded job #{0} (id: {1})", jobNumbers[jobId], jobId));
+                jobsFinishedCount++;
+              }, result.Id);
+
+              remainingJobIds.Remove(result.Id);
+            } else if (result.State == JobState.Aborted) {
+              LogMessage(string.Format("Job #{0} aborted (id: {1})", jobNumbers[result.Id], result.Id));
+              remainingJobIds.Remove(result.Id);
+            } else if (result.State == JobState.Failed) {
+              LogMessage(string.Format("Job {0} failed (id: {1}): {2}", jobNumbers[result.Id], result.Id, result.Exception));
+              remainingJobIds.Remove(result.Id);
+            }
+          }
+        }
+
+        // wait for all tasks to finish downloading and deserializing
+        while (jobsFinishedCount != jobDict.Count) {
+          Thread.Sleep(1000);
+        }
+
+        LogMessage("All jobs finished. Deleting jobs on hive.");
         // delete jobs
         using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
@@ -189 +229 @@
         }
 
-        Log.LogMessage(string.Format("Operations on the hive finished.", jobDict.Count));
+        LogMessage(string.Format("Operations on the hive finished.", jobDict.Count));
       }
       catch (Exception e) {
-        Log.LogException(e);
+        LogException(e);
         throw e;
+      }
+    }
+
+    private void DownloadJob(Guid jobId, IDictionary<IOperation, OperationJob> jobDict, IDictionary<Guid, IOperation> jobIds) {
+      SerializedJob serializedJob;
+      using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
+        serializedJob = service.Obj.GetLastSerializedResult(jobId).Obj;
+      }
+      OperationJob operationJob = SerializedJob.Deserialize<OperationJob>(serializedJob.SerializedJobData);
+      jobDict[jobIds[jobId]] = operationJob;
+      LogMessage(string.Format("Downloaded job (id: {0})", jobId));
+    }
+
+    /// <summary>
+    /// Threadsafe message logging
+    /// </summary>
+    private void LogMessage(string message) {
+      lock (Log) {
+        Log.LogMessage(message);
+      }
+    }
+
+    /// <summary>
+    /// Threadsafe exception logging
+    /// </summary>
+    private void LogException(Exception exception) {
+      lock (Log) {
+        Log.LogException(exception);
       }
     }

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/PreBuildEvent.cmd

@@ -4 +4 @@
 REM SET configfile=f005pc.hagenberg.fhooe.at - app.config
 REM SET configfile=localhost - app.config
-REM SET configfile=services.heuristiclab.com - app.config
-SET configfile=christoph - app.config
+SET configfile=services.heuristiclab.com - app.config
+REM SET configfile=christoph - app.config
 
 IF EXIST "%configfile%" (