Changeset 6394

Timestamp:

06/08/11 14:26:36 (13 years ago)

Author:

svonolfe

Message:

Added first working version of the MPI engine (#1542)

Location:

branches/MPI

Files:

• HeuristicLab.ExtLibs/HeuristicLab.MPInet/1.0.0/HeuristicLab.MPInet-1.0.0.csproj (modified) (1 diff)
• HeuristicLab.MPIAlgorithmRunner/3.3/AlgorithmBroker.cs (modified) (3 diffs)
• HeuristicLab.MPIAlgorithmRunner/3.3/HeuristicLab.MPIAlgorithmRunner-3.3.csproj (modified) (2 diffs)
• HeuristicLab.MPIAlgorithmRunner/3.3/Program.cs (modified) (2 diffs)
• HeuristicLab.MPIEngine/3.3/HeuristicLab.MPIEngine-3.3.csproj (modified) (1 diff)
• HeuristicLab.MPIEngine/3.3/MPIEngine.cs (modified) (8 diffs)
• HeuristicLab.Operators.MPISupport/3.3/MPITransportWrapper.cs (modified) (1 diff)

branches/MPI/HeuristicLab.ExtLibs/HeuristicLab.MPInet/1.0.0/HeuristicLab.MPInet-1.0.0.csproj

@@ -168 +168 @@
   </PropertyGroup>
   <PropertyGroup>
-    <PostBuildEvent>
-    </PostBuildEvent>
+    <PostBuildEvent>set ProjectName=$(ProjectName)
+set Source=$(TargetPath)
+set Configuration=$(ConfigurationName)
+set Platform=$(PlatformName)
+set SolutionDir=$(SolutionDir)
+
+if exist "$(SolutionDir)CustomPostBuild.cmd" call "$(SolutionDir)CustomPostBuild.cmd"</PostBuildEvent>
   </PropertyGroup>
 </Project>

branches/MPI/HeuristicLab.MPIAlgorithmRunner/3.3/AlgorithmBroker.cs

@@ -32 +32 @@
 using HeuristicLab.Core;
 using System.Threading;
+using System.Xml;
                      
 namespace HeuristicLab.MPIAlgorithmRunner {
@@ -53 +54 @@
       endpoint.Name = "AlgorithmBrokerEndpoint";
       NetTcpBinding netTCPBinding = new NetTcpBinding(SecurityMode.None);
+      netTCPBinding.MaxReceivedMessageSize = int.MaxValue;
+      XmlDictionaryReaderQuotas quotas = new XmlDictionaryReaderQuotas();
+      quotas.MaxArrayLength = int.MaxValue;
+      netTCPBinding.ReaderQuotas = quotas;
       endpoint.Binding = netTCPBinding;
       int port = FindFreeTcpPort();
       endpoint.Address = new EndpointAddress("net.tcp://localhost:" + port + "/AlgorithmBroker");
       service.AddServiceEndpoint(endpoint);
+      ServiceDebugBehavior debug = service.Description.Behaviors.Find<ServiceDebugBehavior>();
+      debug.IncludeExceptionDetailInFaults = true;
 
       return service;
@@ -85 +92 @@
     public void TransmitAlgorithm(MPITransportWrapper<IAlgorithm> algorithm) {
       Console.WriteLine("Transmitting Algorithm...");
-      int clients = MPI.Communicator.world.Group.Size;
+      int clients = MPI.Communicator.world.Group.Size - 1;
 
       for(int i = 0; i < clients; i++) {

branches/MPI/HeuristicLab.MPIAlgorithmRunner/3.3/HeuristicLab.MPIAlgorithmRunner-3.3.csproj

@@ -129 +129 @@
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.ServiceModel" />
     <Reference Include="System.Xml.Linq" />
@@ -168 +169 @@
 SubWCRev "%25ProjectDir%25\" "%25ProjectDir%25\HeuristicLabMPIAlgorithmRunnerPlugin.cs.frame" "%25ProjectDir%25\HeuristicLabMPIAlgorithmRunnerPlugin.cs"</PreBuildEvent>
   </PropertyGroup>
+  <PropertyGroup>
+    <PostBuildEvent>set ProjectName=$(ProjectName)
+set Source=$(TargetPath)
+set Configuration=$(ConfigurationName)
+set Platform=$(PlatformName)
+set SolutionDir=$(SolutionDir)
+
+if exist "$(SolutionDir)CustomPostBuild.cmd" call "$(SolutionDir)CustomPostBuild.cmd"</PostBuildEvent>
+  </PropertyGroup>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

branches/MPI/HeuristicLab.MPIAlgorithmRunner/3.3/Program.cs

@@ -41 +41 @@
 
       // Discover the job's context from the environment
-      String headNodeName = System.Environment.GetEnvironmentVariable("CCP_CLUSTER_NAME");
+      String headNodeName = System.Environment.GetEnvironmentVariable("CCP_SCHEDULER");
       int jobId = System.Convert.ToInt32(System.Environment.GetEnvironmentVariable("CCP_JOBID"));
+
+      Console.WriteLine(jobId + "@" + headNodeName); 
 
       // Connect to the head node and get the job
@@ -63 +65 @@
     static void Main(string[] args) {
       using (new MPI.Environment(ref args)) {
-        int clients = MPI.Communicator.world.Group.Size;
+        int clients = MPI.Communicator.world.Group.Size - 1;
+        Console.WriteLine("Clients: " + clients);
+
         int updateInterval = 5000;
 

branches/MPI/HeuristicLab.MPIEngine/3.3/HeuristicLab.MPIEngine-3.3.csproj

@@ -128 +128 @@
       <HintPath>..\..\HeuristicLab.ExtLibs\HeuristicLab.MPInet\MPIAlgorithmRunner\3.3\Microsoft.Hpc.Scheduler.dll</HintPath>
     </Reference>
+    <Reference Include="Microsoft.Hpc.Scheduler.Properties, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\..\Microsoft.Hpc.Scheduler.Properties.dll</HintPath>
+    </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.ServiceModel" />
     <Reference Include="System.Xml.Linq" />

branches/MPI/HeuristicLab.MPIEngine/3.3/MPIEngine.cs

@@ -35 +35 @@
 using HeuristicLab.MPIAlgorithmRunner;
 using HeuristicLab.Operators.MPISupport;
+using Microsoft.Hpc.Scheduler.Properties;
+using System.Xml;
 
 namespace HeuristicLab.MPIEngine {
@@ -74 +76 @@
     }
 
+    protected override void OnPaused() {
+      base.OnPaused();
+
+      Stop();
+    }
+
     protected override void Run(System.Threading.CancellationToken cancellationToken) {
+      string username = @"user";
+      string password = @"password";
+      string headNode = "blade00.hpc.fh-hagenberg.at";
+      List<string> requestedNodes = new List<string>();
+      requestedNodes.Add("BLADE00");
+      string path = @"C:\public\MPISupport";
+      int cpuPerNode = 8;
+
       if (ExecutionStack.Count == 1) {
         ExecutionContext context = ExecutionStack.Pop() as ExecutionContext;
@@ -81 +97 @@
 
         string exec = @"mpiexec";
-        string args = @"-c 3 C:\public\MPISupport\HeuristicLab.MPIAlgorithmRunner-3.3.exe ";
+        string args = @"-c " + cpuPerNode + " /genvlist CCP_JOBID " + path + @"\HeuristicLab.MPIAlgorithmRunner-3.3.exe";
 
         IScheduler scheduler = new Scheduler();
-        scheduler.Connect("blade00.hpc.fh-hagenberg.at");
+        scheduler.Connect(headNode);
 
         ISchedulerJob job = scheduler.CreateJob();
         job.Name = "HeuristicLab.MPIEngine";
-        job.RequestedNodes.Add("BLADE03");
+        foreach (string requestedNode in requestedNodes)
+          job.RequestedNodes.Add(requestedNode);
         ISchedulerTask task = job.CreateTask();
         task.Name = "HeuristicLab.MPIAlgorithmRunner";
@@ -94 +111 @@
         task.StdOutFilePath = "stdout.txt";
         task.StdErrFilePath = "stderr.txt";
+        task.WorkDirectory = path; 
         task.MinimumNumberOfCores = task.MaximumNumberOfCores = 3;
         job.AddTask(task);
 
-        scheduler.SubmitJob(job, @"HPC\svonolfe", @"Vunkopf!3");
+        scheduler.SubmitJob(job, username, "");
 
         try {
@@ -104 +122 @@
 
           while (address == null && timeout > 0) {
+            cancellationToken.ThrowIfCancellationRequested();
+
             ISchedulerJob schedulerJob = scheduler.OpenJob(job.Id);
             if (schedulerJob != null) {
-              INameValueCollection properties = schedulerJob.GetCustomProperties();
-              NameValue item = properties.FirstOrDefault(i => i.Name == "address");
-              if (item != null) {
-                address = item.Value;
+              NameValue property = schedulerJob.GetCustomProperties().FirstOrDefault(p => p.Name == "address");
+
+              if (property != null) {
+                address = property.Value;
               } else {
                 System.Threading.Thread.Sleep(1000);
                 timeout--;
               }
-            }
+            }            
           }
 
@@ -122 +142 @@
 
           NetTcpBinding netTCPBinding = new NetTcpBinding(SecurityMode.None);
+          XmlDictionaryReaderQuotas quotas = new XmlDictionaryReaderQuotas();
+          quotas.MaxArrayLength = int.MaxValue;
+          netTCPBinding.ReaderQuotas = quotas;
+          netTCPBinding.MaxReceivedMessageSize = int.MaxValue;
           ChannelFactory<IAlgorithmBroker> factory = new ChannelFactory<IAlgorithmBroker>(netTCPBinding, address);
           IAlgorithmBroker proxy = factory.CreateChannel();
@@ -128 +152 @@
 
           while (!proxy.IsAlgorithmTerminated()) {
+            cancellationToken.ThrowIfCancellationRequested();
+
             ItemList<ResultCollection> results = proxy.GetResults().InnerItem;
 
@@ -143 +169 @@
         }
         catch (Exception e) {
-          scheduler.CancelJob(job.Id, "Exception: " + e.Message);
+          scheduler.CancelJob(job.Id, "Exception: " + e.GetType());
           throw e;
+        }
+        finally {
+           /*ISchedulerJob schedulerJob = scheduler.OpenJob(job.Id);
+           if (schedulerJob != null &&
+             (schedulerJob.State == JobState.Running || schedulerJob.State == JobState.Queued)) {
+               scheduler.CancelJob(job.Id, "Cancelled");            
+           } */
         }
       }

branches/MPI/HeuristicLab.Operators.MPISupport/3.3/MPITransportWrapper.cs

@@ -49 +49 @@
 
     protected MPITransportWrapper(SerializationInfo info, StreamingContext context) {
-      MemoryStream stream = new MemoryStream(info.GetValue("Item", typeof(byte[])) as byte[]);
-      if (stream != null)
-        innerItem = XmlParser.Deserialize(stream) as T;
+      byte[] buffer = info.GetValue("Item", typeof(byte[])) as byte[];
+      if (buffer != null)
+        innerItem = XmlParser.Deserialize(new MemoryStream(buffer)) as T;
       else
         innerItem = null;