Changeset 2073

Timestamp:

06/19/09 15:21:33 (15 years ago)

Author:

gkronber

Message:

Fixed bugs in preparation of engines for execution on hive. Used HL.Tracing instead of trace statements. #642 (Hive backend for CEDMA)

Location:

trunk/sources

Files:

• HeuristicLab.CEDMA.Server/3.3/GridExecuter.cs (modified) (2 diffs)
• HeuristicLab.Grid.HiveBridge/3.2/HeuristicLab.Grid.HiveBridge-3.2.csproj (modified) (1 diff)
• HeuristicLab.Grid.HiveBridge/3.2/HeuristicLabGridHiveBridgePlugin.cs (modified) (1 diff)
• HeuristicLab.Grid.HiveBridge/3.2/HiveGridServerWrapper.cs (modified) (5 diffs)
• HeuristicLab.Grid/3.2/JobManager.cs (modified) (3 diffs)
• HeuristicLab.Grid/3.2/ProcessingEngine.cs (modified) (2 diffs)
• HeuristicLab.sln (modified) (1 diff)

trunk/sources/HeuristicLab.CEDMA.Server/3.3/GridExecuter.cs

@@ -74 +74 @@
               procEngine.OperatorGraph.AddOperator(algorithm.Engine.OperatorGraph.InitialOperator);
               procEngine.OperatorGraph.InitialOperator = algorithm.Engine.OperatorGraph.InitialOperator;
+              procEngine.Reset();
               AsyncGridResult asyncResult = jobManager.BeginExecuteEngine(procEngine);
               asyncResults.Add(asyncResult.WaitHandle, asyncResult);
@@ -100 +101 @@
             }
             catch (Exception badEx) {
-              Trace.WriteLine("CEDMA Executer: Exception in job execution thread. " + badEx.Message);
+              HeuristicLab.Tracing.Logger.Error("CEDMA Executer: Exception in job execution thread. " + badEx.Message);
             }
           }
         }
         catch (Exception ex) {
-          Trace.WriteLine("CEDMA Executer: Exception in job-management thread. " + ex.Message);
+          HeuristicLab.Tracing.Logger.Warn("CEDMA Executer: Exception in job-management thread. " + ex.Message);
         }
       }

trunk/sources/HeuristicLab.Grid.HiveBridge/3.2/HeuristicLab.Grid.HiveBridge-3.2.csproj

@@ -101 +101 @@
       <Name>HeuristicLab.PluginInfrastructure</Name>
     </ProjectReference>
+    <ProjectReference Include="..\..\HeuristicLab.Tracing\3.2\HeuristicLab.Tracing-3.2.csproj">
+      <Project>{EE2034D9-6E27-48A1-B855-42D45F69A4FC}</Project>
+      <Name>HeuristicLab.Tracing-3.2</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>

trunk/sources/HeuristicLab.Grid.HiveBridge/3.2/HeuristicLabGridHiveBridgePlugin.cs

@@ -33 +33 @@
   [Dependency(Dependency = "HeuristicLab.Hive.JobBase-3.2")]
   [Dependency(Dependency = "HeuristicLab.Hive.Engine-3.2")]
+  [Dependency(Dependency = "HeuristicLab.Tracing-3.2")]
   public class HeuristicLabGridPlugin : PluginBase {
   }

trunk/sources/HeuristicLab.Grid.HiveBridge/3.2/HiveGridServerWrapper.cs

@@ -32 +32 @@
 using HeuristicLab.PluginInfrastructure;
 using HeuristicLab.Hive.Contracts.BusinessObjects;
+using System.ServiceModel;
+using HeuristicLab.Tracing;
 
 namespace HeuristicLab.Grid.HiveBridge {
   public class HiveGridServerWrapper : IGridServer {
+    private const int MAX_CONNECTION_RETRIES = 10;
+    private const int RETRY_TIMEOUT_SEC = 60;
     private string address;
+    private IExecutionEngineFacade executionEngine;
+    private object connectionLock = new object();
 
     public HiveGridServerWrapper(string address) {
@@ -42 +48 @@
 
     public JobState JobState(Guid guid) {
-      IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(address);
-      ResponseObject<JobResult> response = executionEngineFacade.GetLastResult(guid, false);
-      if (response.Success == true &&
+      ResponseObject<JobResult> response = SavelyExecute(() => executionEngine.GetLastResult(guid, false));
+      if (response != null && response.Success == true &&
         (response.StatusMessage == ApplicationConstants.RESPONSE_JOB_RESULT_NOT_YET_HERE ||
           response.StatusMessage == ApplicationConstants.RESPONSE_JOB_REQUEST_SET ||
           response.StatusMessage == ApplicationConstants.RESPONSE_JOB_REQUEST_ALLREADY_SET ||
           response.StatusMessage == ApplicationConstants.RESPONSE_JOB_JOB_RESULT_SENT)) {
-          return HeuristicLab.Grid.JobState.Busy;
+        return HeuristicLab.Grid.JobState.Busy;
       } else return HeuristicLab.Grid.JobState.Unknown;
     }
@@ -56 +61 @@
       var jobObj = CreateJobObj(engine);
 
-      IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(address);
-      ResponseObject<HeuristicLab.Hive.Contracts.BusinessObjects.Job> res = executionEngineFacade.AddJob(jobObj);
-      return res.Obj.Id;
+      ResponseObject<HeuristicLab.Hive.Contracts.BusinessObjects.Job> res = SavelyExecute(() => executionEngine.AddJob(jobObj));
+      return res == null ? Guid.Empty : res.Obj.Id;
     }
 
     public byte[] TryEndExecuteEngine(Guid guid) {
-      IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(address);
-      ResponseObject<JobResult> response = executionEngineFacade.GetLastResult(guid, false);
-      if (response.Success && response.Obj != null) {
+      ResponseObject<JobResult> response = SavelyExecute(() => executionEngine.GetLastResult(guid, false));
+      if (response != null &&
+        response.Success && response.Obj != null) {
         HeuristicLab.Hive.Engine.Job restoredJob = (HeuristicLab.Hive.Engine.Job)PersistenceManager.RestoreFromGZip(response.Obj.Result);
         // Serialize the engine
@@ -89 +93 @@
       job.Engine.OperatorGraph.AddOperator(engine.OperatorGraph.InitialOperator);
       job.Engine.OperatorGraph.InitialOperator = engine.OperatorGraph.InitialOperator;
+      job.Engine.Reset();
 
       // Serialize the job
@@ -143 +148 @@
       return engine;
     }
+
+    private TResult SavelyExecute<TResult>(Func<TResult> a) where TResult : Response {
+      int retries = 0;
+      if (executionEngine == null)
+        executionEngine = ServiceLocator.CreateExecutionEngineFacade(address);
+
+      do {
+        try {
+          lock (connectionLock) {
+            return a();
+          }
+        }
+        catch (TimeoutException) {
+          retries++;
+          Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
+        }
+        catch (CommunicationException) {
+          executionEngine = ServiceLocator.CreateExecutionEngineFacade(address);
+          retries++;
+          Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
+        }
+      } while (retries < MAX_CONNECTION_RETRIES);
+      Logger.Warn("Reached max connection retries");
+      return null;
+    }
   }
 }

trunk/sources/HeuristicLab.Grid/3.2/JobManager.cs

@@ -42 +42 @@
 
     private IGridServer server;
-    private string address;
     private object waitingQueueLock = new object();
     private Queue<AsyncGridResult> waitingJobs = new Queue<AsyncGridResult>();
@@ -108 +107 @@
       }
       catch (Exception e) {
-        Trace.TraceError("Exception " + e + " in JobManager.StartEngines() killed the start-engine thread\n" + e.StackTrace);
+        HeuristicLab.Tracing.Logger.Error("Exception " + e + " in JobManager.StartEngines() killed the start-engine thread\n" + e.StackTrace);
       }
     }
@@ -150 +149 @@
       }
       catch (Exception e) {
-        Trace.TraceError("Exception " + e + " in JobManager.GetResults() killed the results-gathering thread\n" + e.StackTrace);
+        HeuristicLab.Tracing.Logger.Error("Exception " + e + " in JobManager.GetResults() killed the results-gathering thread\n" + e.StackTrace);
       }
     }

trunk/sources/HeuristicLab.Grid/3.2/ProcessingEngine.cs

@@ -27 +27 @@
 using System.Xml;
 using System.Threading;
-using System.Diagnostics;
 
 namespace HeuristicLab.Grid {
@@ -82 +81 @@
         } catch(Exception ex) {
           errorMessage = CreateErrorMessage(ex);
-          Trace.TraceWarning(errorMessage);
+          HeuristicLab.Tracing.HiveLogger.Error(errorMessage);
           // push operation on stack again
           myExecutionStack.Push(atomicOperation);

trunk/sources/HeuristicLab.sln

@@ -3862 +3862 @@
     {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|Any CPU.Build.0 = Debug|Any CPU
     {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x64.ActiveCfg = Debug|Any CPU
-    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x86.ActiveCfg = Debug|Any CPU
+    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x86.ActiveCfg = Debug|x86
+    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x86.Build.0 = Debug|x86
     {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
     {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|Any CPU.Build.0 = Debug|Any CPU