Changeset 2073


Ignore:
Timestamp:
06/19/09 15:21:33 (12 years ago)
Author:
gkronber
Message:

Fixed bugs in preparation of engines for execution on hive. Used HL.Tracing instead of trace statements. #642 (Hive backend for CEDMA)

Location:
trunk/sources
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • trunk/sources/HeuristicLab.CEDMA.Server/3.3/GridExecuter.cs

    r2064 r2073  
    7474              procEngine.OperatorGraph.AddOperator(algorithm.Engine.OperatorGraph.InitialOperator);
    7575              procEngine.OperatorGraph.InitialOperator = algorithm.Engine.OperatorGraph.InitialOperator;
     76              procEngine.Reset();
    7677              AsyncGridResult asyncResult = jobManager.BeginExecuteEngine(procEngine);
    7778              asyncResults.Add(asyncResult.WaitHandle, asyncResult);
     
    100101            }
    101102            catch (Exception badEx) {
    102               Trace.WriteLine("CEDMA Executer: Exception in job execution thread. " + badEx.Message);
     103              HeuristicLab.Tracing.Logger.Error("CEDMA Executer: Exception in job execution thread. " + badEx.Message);
    103104            }
    104105          }
    105106        }
    106107        catch (Exception ex) {
    107           Trace.WriteLine("CEDMA Executer: Exception in job-management thread. " + ex.Message);
     108          HeuristicLab.Tracing.Logger.Warn("CEDMA Executer: Exception in job-management thread. " + ex.Message);
    108109        }
    109110      }
  • trunk/sources/HeuristicLab.Grid.HiveBridge/3.2/HeuristicLab.Grid.HiveBridge-3.2.csproj

    r2064 r2073  
    101101      <Name>HeuristicLab.PluginInfrastructure</Name>
    102102    </ProjectReference>
     103    <ProjectReference Include="..\..\HeuristicLab.Tracing\3.2\HeuristicLab.Tracing-3.2.csproj">
     104      <Project>{EE2034D9-6E27-48A1-B855-42D45F69A4FC}</Project>
     105      <Name>HeuristicLab.Tracing-3.2</Name>
     106    </ProjectReference>
    103107  </ItemGroup>
    104108  <ItemGroup>
  • trunk/sources/HeuristicLab.Grid.HiveBridge/3.2/HeuristicLabGridHiveBridgePlugin.cs

    r2064 r2073  
    3333  [Dependency(Dependency = "HeuristicLab.Hive.JobBase-3.2")]
    3434  [Dependency(Dependency = "HeuristicLab.Hive.Engine-3.2")]
     35  [Dependency(Dependency = "HeuristicLab.Tracing-3.2")]
    3536  public class HeuristicLabGridPlugin : PluginBase {
    3637  }
  • trunk/sources/HeuristicLab.Grid.HiveBridge/3.2/HiveGridServerWrapper.cs

    r2064 r2073  
    3232using HeuristicLab.PluginInfrastructure;
    3333using HeuristicLab.Hive.Contracts.BusinessObjects;
     34using System.ServiceModel;
     35using HeuristicLab.Tracing;
    3436
    3537namespace HeuristicLab.Grid.HiveBridge {
    3638  public class HiveGridServerWrapper : IGridServer {
     39    private const int MAX_CONNECTION_RETRIES = 10;
     40    private const int RETRY_TIMEOUT_SEC = 60;
    3741    private string address;
     42    private IExecutionEngineFacade executionEngine;
     43    private object connectionLock = new object();
    3844
    3945    public HiveGridServerWrapper(string address) {
     
    4248
    4349    public JobState JobState(Guid guid) {
    44       IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(address);
    45       ResponseObject<JobResult> response = executionEngineFacade.GetLastResult(guid, false);
    46       if (response.Success == true &&
     50      ResponseObject<JobResult> response = SavelyExecute(() => executionEngine.GetLastResult(guid, false));
     51      if (response != null && response.Success == true &&
    4752        (response.StatusMessage == ApplicationConstants.RESPONSE_JOB_RESULT_NOT_YET_HERE ||
    4853          response.StatusMessage == ApplicationConstants.RESPONSE_JOB_REQUEST_SET ||
    4954          response.StatusMessage == ApplicationConstants.RESPONSE_JOB_REQUEST_ALLREADY_SET ||
    5055          response.StatusMessage == ApplicationConstants.RESPONSE_JOB_JOB_RESULT_SENT)) {
    51           return HeuristicLab.Grid.JobState.Busy;
     56        return HeuristicLab.Grid.JobState.Busy;
    5257      } else return HeuristicLab.Grid.JobState.Unknown;
    5358    }
     
    5661      var jobObj = CreateJobObj(engine);
    5762
    58       IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(address);
    59       ResponseObject<HeuristicLab.Hive.Contracts.BusinessObjects.Job> res = executionEngineFacade.AddJob(jobObj);
    60       return res.Obj.Id;
     63      ResponseObject<HeuristicLab.Hive.Contracts.BusinessObjects.Job> res = SavelyExecute(() => executionEngine.AddJob(jobObj));
     64      return res == null ? Guid.Empty : res.Obj.Id;
    6165    }
    6266
    6367    public byte[] TryEndExecuteEngine(Guid guid) {
    64       IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(address);
    65       ResponseObject<JobResult> response = executionEngineFacade.GetLastResult(guid, false);
    66       if (response.Success && response.Obj != null) {
     68      ResponseObject<JobResult> response = SavelyExecute(() => executionEngine.GetLastResult(guid, false));
     69      if (response != null &&
     70        response.Success && response.Obj != null) {
    6771        HeuristicLab.Hive.Engine.Job restoredJob = (HeuristicLab.Hive.Engine.Job)PersistenceManager.RestoreFromGZip(response.Obj.Result);
    6872        // Serialize the engine
     
    8993      job.Engine.OperatorGraph.AddOperator(engine.OperatorGraph.InitialOperator);
    9094      job.Engine.OperatorGraph.InitialOperator = engine.OperatorGraph.InitialOperator;
     95      job.Engine.Reset();
    9196
    9297      // Serialize the job
     
    143148      return engine;
    144149    }
     150
     151    private TResult SavelyExecute<TResult>(Func<TResult> a) where TResult : Response {
     152      int retries = 0;
     153      if (executionEngine == null)
     154        executionEngine = ServiceLocator.CreateExecutionEngineFacade(address);
     155
     156      do {
     157        try {
     158          lock (connectionLock) {
     159            return a();
     160          }
     161        }
     162        catch (TimeoutException) {
     163          retries++;
     164          Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
     165        }
     166        catch (CommunicationException) {
     167          executionEngine = ServiceLocator.CreateExecutionEngineFacade(address);
     168          retries++;
     169          Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
     170        }
     171      } while (retries < MAX_CONNECTION_RETRIES);
     172      Logger.Warn("Reached max connection retries");
     173      return null;
     174    }
    145175  }
    146176}
  • trunk/sources/HeuristicLab.Grid/3.2/JobManager.cs

    r2058 r2073  
    4242
    4343    private IGridServer server;
    44     private string address;
    4544    private object waitingQueueLock = new object();
    4645    private Queue<AsyncGridResult> waitingJobs = new Queue<AsyncGridResult>();
     
    108107      }
    109108      catch (Exception e) {
    110         Trace.TraceError("Exception " + e + " in JobManager.StartEngines() killed the start-engine thread\n" + e.StackTrace);
     109        HeuristicLab.Tracing.Logger.Error("Exception " + e + " in JobManager.StartEngines() killed the start-engine thread\n" + e.StackTrace);
    111110      }
    112111    }
     
    150149      }
    151150      catch (Exception e) {
    152         Trace.TraceError("Exception " + e + " in JobManager.GetResults() killed the results-gathering thread\n" + e.StackTrace);
     151        HeuristicLab.Tracing.Logger.Error("Exception " + e + " in JobManager.GetResults() killed the results-gathering thread\n" + e.StackTrace);
    153152      }
    154153    }
  • trunk/sources/HeuristicLab.Grid/3.2/ProcessingEngine.cs

    r1529 r2073  
    2727using System.Xml;
    2828using System.Threading;
    29 using System.Diagnostics;
    3029
    3130namespace HeuristicLab.Grid {
     
    8281        } catch(Exception ex) {
    8382          errorMessage = CreateErrorMessage(ex);
    84           Trace.TraceWarning(errorMessage);
     83          HeuristicLab.Tracing.HiveLogger.Error(errorMessage);
    8584          // push operation on stack again
    8685          myExecutionStack.Push(atomicOperation);
  • trunk/sources/HeuristicLab.sln

    r2058 r2073  
    38623862    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|Any CPU.Build.0 = Debug|Any CPU
    38633863    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x64.ActiveCfg = Debug|Any CPU
    3864     {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x86.ActiveCfg = Debug|Any CPU
     3864    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     3865    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x86.Build.0 = Debug|x86
    38653866    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    38663867    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|Any CPU.Build.0 = Debug|Any CPU
Note: See TracChangeset for help on using the changeset viewer.