Free cookie consent management tool by TermsFeed Policy Generator

Changeset 2058


Ignore:
Timestamp:
06/18/09 15:01:10 (15 years ago)
Author:
gkronber
Message:

Refactored JobManager and added a plugin that contains a bridge between grid and hive. The bridge allows to use the execution engine service of Hive as a grid server. This way CEDMA job execution and DistributedEngine job execution can either use Hive or Grid as backend. #642 (Hive backend for CEDMA)

Location:
trunk/sources
Files:
9 added
1 deleted
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/sources/HeuristicLab.CEDMA.Server/3.3/GridExecuter.cs

    r2057 r2058  
    5353    }
    5454
    55     public GridExecuter(IDispatcher dispatcher, IStore store, string gridUrl)
     55    public GridExecuter(IDispatcher dispatcher, IStore store, IGridServer server)
    5656      : base(dispatcher, store) {
    57       this.jobManager = new JobManager(gridUrl);
     57      this.jobManager = new JobManager(server);
    5858      activeAlgorithms = new Dictionary<AsyncGridResult, IAlgorithm>();
    5959      jobManager.Reset();

  • trunk/sources/HeuristicLab.CEDMA.Server/3.3/HeuristicLab.CEDMA.Server-3.3.csproj

    r2057 r2058  
    9292  <ItemGroup>
    9393    <Compile Include="ExecuterBase.cs" />
    94     <Compile Include="HiveExecuter.cs">
    95       <SubType>Code</SubType>
    96     </Compile>
    9794    <Compile Include="IExecuter.cs" />
    9895    <Compile Include="DispatcherBase.cs" />
    … …  
    136133      <Name>HeuristicLab.Data-3.2</Name>
    137134    </ProjectReference>
     135    <ProjectReference Include="..\..\HeuristicLab.Grid.HiveBridge\3.2\HeuristicLab.Grid.HiveBridge-3.2.csproj">
     136      <Project>{DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}</Project>
     137      <Name>HeuristicLab.Grid.HiveBridge-3.2</Name>
     138    </ProjectReference>
    138139    <ProjectReference Include="..\..\HeuristicLab.Grid\3.2\HeuristicLab.Grid-3.2.csproj">
    139140      <Project>{545CE756-98D8-423B-AC2E-6E7D70926E5C}</Project>
    140141      <Name>HeuristicLab.Grid-3.2</Name>
    141     </ProjectReference>
    142     <ProjectReference Include="..\..\HeuristicLab.Hive.Engine\3.2\HeuristicLab.Hive.Engine-3.2.csproj">
    143       <Project>{C8FEDAC1-0326-4293-B585-F0FEDDEDFC11}</Project>
    144       <Name>HeuristicLab.Hive.Engine-3.2</Name>
    145142    </ProjectReference>
    146143    <ProjectReference Include="..\..\HeuristicLab.Modeling\3.2\HeuristicLab.Modeling-3.2.csproj">

  • trunk/sources/HeuristicLab.CEDMA.Server/3.3/HeuristicLabCedmaServerPlugin.cs

    r2057 r2058  
    3131  [Dependency(Dependency = "HeuristicLab.CEDMA.DB-3.3")]
    3232  [Dependency(Dependency = "HeuristicLab.Grid-3.2")]
     33  [Dependency(Dependency = "HeuristicLab.Grid.HiveBridge-3.2")]
    3334  [Dependency(Dependency = "HeuristicLab.Core-3.2")]
    3435  [Dependency(Dependency = "HeuristicLab.Data-3.2")]

  • trunk/sources/HeuristicLab.CEDMA.Server/3.3/Server.cs

    r2050 r2058  
    3030using HeuristicLab.CEDMA.DB;
    3131using System.ServiceModel.Description;
     32using HeuristicLab.Grid;
     33using HeuristicLab.Grid.HiveBridge;
    3234
    3335namespace HeuristicLab.CEDMA.Server {
    … …  
    105107    internal void Connect(string serverUrl) {
    106108      dispatcher = new SimpleDispatcher(store);
     109      IGridServer gridServer = null;
    107110      if (serverUrl.Contains("ExecutionEngine")) {
    108         executer = new HiveExecuter(dispatcher, store, serverUrl);
     111        gridServer = new HiveGridServerWrapper(serverUrl);
    109112      } else {
    110113        // default is grid backend
    111         executer = new GridExecuter(dispatcher, store, serverUrl);
     114        gridServer = new GridServerProxy(serverUrl);
    112115      }
     116      executer = new GridExecuter(dispatcher, store, gridServer);
    113117      executer.Start();
    114118    }

  • trunk/sources/HeuristicLab.DistributedEngine/3.2/DistributedEngine.cs

    r2055 r2058  
    7272
    7373    public override void Execute() {
    74       if (jobManager == null) this.jobManager = new JobManager(serverAddress);
     74      if (jobManager == null) this.jobManager = new JobManager(new GridServerProxy(serverAddress));
    7575      jobManager.Reset();
    7676      base.Execute();

  • trunk/sources/HeuristicLab.Grid/3.2/HeuristicLab.Grid-3.2.csproj

    r2055 r2058  
    9999    </Compile>
    100100    <Compile Include="AsyncGridResult.cs" />
     101    <Compile Include="GridServerProxy.cs" />
    101102    <Compile Include="Database.cs" />
    102103    <Compile Include="EngineRunner.cs" />
    … …  
    128129      <Project>{94186A6A-5176-4402-AE83-886557B53CCA}</Project>
    129130      <Name>HeuristicLab.PluginInfrastructure</Name>
     131    </ProjectReference>
     132    <ProjectReference Include="..\..\HeuristicLab.Tracing\3.2\HeuristicLab.Tracing-3.2.csproj">
     133      <Project>{EE2034D9-6E27-48A1-B855-42D45F69A4FC}</Project>
     134      <Name>HeuristicLab.Tracing-3.2</Name>
    130135    </ProjectReference>
    131136  </ItemGroup>

  • trunk/sources/HeuristicLab.Grid/3.2/HeuristicLabGridPlugin.cs

    r1529 r2058  
    3030  [PluginFile(Filename = "HeuristicLab.Grid-3.2.dll", Filetype = PluginFileType.Assembly)]
    3131  [Dependency(Dependency = "HeuristicLab.Core-3.2")]
     32  [Dependency(Dependency = "HeuristicLab.Tracing-3.2")]
    3233  public class HeuristicLabGridPlugin : PluginBase {
    3334  }

  • trunk/sources/HeuristicLab.Grid/3.2/JobManager.cs

    r2055 r2058  
    3939  public class JobManager {
    4040    private const int MAX_RESTARTS = 5;
    41     private const int MAX_CONNECTION_RETRIES = 10;
    42     private const int RETRY_TIMEOUT_SEC = 60;
    4341    private const int RESULT_POLLING_TIMEOUT = 5;
    4442
    … …  
    4947    private object runningQueueLock = new object();
    5048    private Queue<AsyncGridResult> runningJobs = new Queue<AsyncGridResult>();
    51     private object connectionLock = new object();
    52 
    5349    private AutoResetEvent runningWaitHandle = new AutoResetEvent(false);
    5450    private AutoResetEvent waitingWaitHandle = new AutoResetEvent(false);
    5551
    56     private ChannelFactory<IGridServer> factory;
    5752
    58     public JobManager(string address) {
    59       this.address = address;
     53    public JobManager(IGridServer server) {
     54      this.server = server;
    6055      Thread starterThread = new Thread(StartEngines);
    6156      Thread resultsGatheringThread = new Thread(GetResults);
    … …  
    6560
    6661    public void Reset() {
    67       ResetConnection();
    6862      lock (waitingQueueLock) {
    6963        foreach (AsyncGridResult r in waitingJobs) {
    … …  
    8074    }
    8175
    82     private void ResetConnection() {
    83       Trace.TraceInformation("Reset connection in JobManager");
    84       lock (connectionLock) {
    85         // open a new channel
    86         NetTcpBinding binding = new NetTcpBinding();
    87         binding.MaxReceivedMessageSize = 100000000; // 100Mbytes
    88         binding.ReaderQuotas.MaxStringContentLength = 100000000; // also 100M chars
    89         binding.ReaderQuotas.MaxArrayLength = 100000000; // also 100M elements;
    90 
    91         factory = new ChannelFactory<IGridServer>(binding);
    92         server = factory.CreateChannel(new EndpointAddress(address));
    93       }
    94     }
    9576
    9677    public void StartEngines() {
    … …  
    10384          if (job == null) waitingWaitHandle.WaitOne(); // no jobs waiting
    10485          else {
    105             Guid currentEngineGuid = TryStartExecuteEngine(job.Engine);
     86            Guid currentEngineGuid = server.BeginExecuteEngine(ZipEngine(job.Engine));
    10687            if (currentEngineGuid == Guid.Empty) {
    10788              // couldn't start the job -> requeue
    … …  
    141122          if (job == null) runningWaitHandle.WaitOne(); // no jobs running
    142123          else {
    143             byte[] zippedResult = TryEndExecuteEngine(server, job.Guid);
     124            byte[] zippedResult = server.TryEndExecuteEngine(job.Guid);
    144125            if (zippedResult != null) {
    145126              // successful => store result
    … …  
    149130            } else {
    150131              // there was a problem -> check the state of the job and restart if necessary
    151               JobState jobState = TryGetJobState(server, job.Guid);
     132              JobState jobState = server.JobState(job.Guid);
    152133              if (jobState == JobState.Unknown) {
    153134                job.Restarts++;
    … …  
    195176      }
    196177    }
    197 
    198     private Guid TryStartExecuteEngine(IEngine engine) {
    199       byte[] zippedEngine = ZipEngine(engine);
    200       return SavelyExecute(() => server.BeginExecuteEngine(zippedEngine));
    201     }
    202 
    203     private byte[] TryEndExecuteEngine(IGridServer server, Guid engineGuid) {
    204       return SavelyExecute(() => {
    205         byte[] zippedResult = server.TryEndExecuteEngine(engineGuid);
    206         return zippedResult;
    207       });
    208     }
    209 
    210     private JobState TryGetJobState(IGridServer server, Guid engineGuid) {
    211       return SavelyExecute(() => server.JobState(engineGuid));
    212     }
    213 
    214     private TResult SavelyExecute<TResult>(Func<TResult> a) {
    215       int retries = 0;
    216       do {
    217         try {
    218           lock (connectionLock) {
    219             return a();
    220           }
    221         }
    222         catch (TimeoutException) {
    223           retries++;
    224           Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
    225         }
    226         catch (CommunicationException) {
    227           ResetConnection();
    228           retries++;
    229           Thread.Sleep(TimeSpan.FromSeconds(RETRY_TIMEOUT_SEC));
    230         }
    231       } while (retries < MAX_CONNECTION_RETRIES);
    232       Trace.TraceWarning("Reached max connection retries");
    233       return default(TResult);
    234     }
    235178  }
    236179}

  • trunk/sources/HeuristicLab.sln

    r2035 r2058  
    230230EndProject
    231231Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Modeling-3.3", "HeuristicLab.Modeling\3.3\HeuristicLab.Modeling-3.3.csproj", "{3A5ACA94-47A7-47D6-B644-2B8A7503FCCC}"
     232EndProject
     233Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Grid.HiveBridge-3.2", "HeuristicLab.Grid.HiveBridge\3.2\HeuristicLab.Grid.HiveBridge-3.2.csproj", "{DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}"
    232234EndProject
    233235Global
    … …  
    976978    {E66D1EAA-525C-4F6D-BD25-DBCAE3750E25}.CEDMA Debug|x64.Build.0 = Debug|x64
    977979    {E66D1EAA-525C-4F6D-BD25-DBCAE3750E25}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     980    {E66D1EAA-525C-4F6D-BD25-DBCAE3750E25}.CEDMA Debug|x86.Build.0 = Debug|x86
    978981    {E66D1EAA-525C-4F6D-BD25-DBCAE3750E25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    979982    {E66D1EAA-525C-4F6D-BD25-DBCAE3750E25}.Debug|Any CPU.Build.0 = Debug|Any CPU
    … …  
    11811184    {74223A32-C726-4978-BE78-37113A18373C}.CEDMA Debug|x64.Build.0 = Debug|x64
    11821185    {74223A32-C726-4978-BE78-37113A18373C}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     1186    {74223A32-C726-4978-BE78-37113A18373C}.CEDMA Debug|x86.Build.0 = Debug|x86
    11831187    {74223A32-C726-4978-BE78-37113A18373C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    11841188    {74223A32-C726-4978-BE78-37113A18373C}.Debug|Any CPU.Build.0 = Debug|Any CPU
    … …  
    12161220    {7C20D100-8BEB-433A-9499-F075E2CB9297}.CEDMA Debug|x64.Build.0 = Debug|x64
    12171221    {7C20D100-8BEB-433A-9499-F075E2CB9297}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     1222    {7C20D100-8BEB-433A-9499-F075E2CB9297}.CEDMA Debug|x86.Build.0 = Debug|x86
    12181223    {7C20D100-8BEB-433A-9499-F075E2CB9297}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    12191224    {7C20D100-8BEB-433A-9499-F075E2CB9297}.Debug|Any CPU.Build.0 = Debug|Any CPU
    … …  
    12511256    {6084CFB5-733F-449D-9F92-2E40D13F0514}.CEDMA Debug|x64.Build.0 = Debug|x64
    12521257    {6084CFB5-733F-449D-9F92-2E40D13F0514}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     1258    {6084CFB5-733F-449D-9F92-2E40D13F0514}.CEDMA Debug|x86.Build.0 = Debug|x86
    12531259    {6084CFB5-733F-449D-9F92-2E40D13F0514}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    12541260    {6084CFB5-733F-449D-9F92-2E40D13F0514}.Debug|Any CPU.Build.0 = Debug|Any CPU
    … …  
    12861292    {545CE756-98D8-423B-AC2E-6E7D70926E5C}.CEDMA Debug|x64.Build.0 = Debug|x64
    12871293    {545CE756-98D8-423B-AC2E-6E7D70926E5C}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     1294    {545CE756-98D8-423B-AC2E-6E7D70926E5C}.CEDMA Debug|x86.Build.0 = Debug|x86
    12881295    {545CE756-98D8-423B-AC2E-6E7D70926E5C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    12891296    {545CE756-98D8-423B-AC2E-6E7D70926E5C}.Debug|Any CPU.Build.0 = Debug|Any CPU
    … …  
    18421849    {4095C92C-5A4C-44BC-9963-5F384CF5CC3F}.Modeling Debug|x64.Build.0 = Debug|x64
    18431850    {4095C92C-5A4C-44BC-9963-5F384CF5CC3F}.Modeling Debug|x86.ActiveCfg = Debug|x86
    1844     {4095C92C-5A4C-44BC-9963-5F384CF5CC3F}.Modeling Debug|x86.Build.0 = Debug|x86
    18451851    {4095C92C-5A4C-44BC-9963-5F384CF5CC3F}.Release|Any CPU.ActiveCfg = Release|Any CPU
    18461852    {4095C92C-5A4C-44BC-9963-5F384CF5CC3F}.Release|Any CPU.Build.0 = Release|Any CPU
    … …  
    33433349    {7E18E25A-BCEB-467A-BACC-ABCA73F9D655}.CEDMA Debug|x64.ActiveCfg = Debug|Any CPU
    33443350    {7E18E25A-BCEB-467A-BACC-ABCA73F9D655}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     3351    {7E18E25A-BCEB-467A-BACC-ABCA73F9D655}.CEDMA Debug|x86.Build.0 = Debug|x86
    33453352    {7E18E25A-BCEB-467A-BACC-ABCA73F9D655}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    33463353    {7E18E25A-BCEB-467A-BACC-ABCA73F9D655}.Debug|Any CPU.Build.0 = Debug|Any CPU
    … …  
    34013408    {80F7FADA-549D-4151-8856-79B620A50DBA}.CEDMA Debug|x64.ActiveCfg = Debug|Any CPU
    34023409    {80F7FADA-549D-4151-8856-79B620A50DBA}.CEDMA Debug|x86.ActiveCfg = Debug|x86
     3410    {80F7FADA-549D-4151-8856-79B620A50DBA}.CEDMA Debug|x86.Build.0 = Debug|x86
    34033411    {80F7FADA-549D-4151-8856-79B620A50DBA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
    34043412    {80F7FADA-549D-4151-8856-79B620A50DBA}.Debug|Any CPU.Build.0 = Debug|Any CPU
    … …  
    38513859    {3A5ACA94-47A7-47D6-B644-2B8A7503FCCC}.Visualization Debug|x86.ActiveCfg = Debug|x86
    38523860    {3A5ACA94-47A7-47D6-B644-2B8A7503FCCC}.Visualization Debug|x86.Build.0 = Debug|x86
     3861    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|Any CPU.ActiveCfg = Debug|Any CPU
     3862    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|Any CPU.Build.0 = Debug|Any CPU
     3863    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x64.ActiveCfg = Debug|Any CPU
     3864    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.CEDMA Debug|x86.ActiveCfg = Debug|Any CPU
     3865    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
     3866    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|Any CPU.Build.0 = Debug|Any CPU
     3867    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|x64.ActiveCfg = Debug|Any CPU
     3868    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|x86.ActiveCfg = Debug|x86
     3869    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Debug|x86.Build.0 = Debug|x86
     3870    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Modeling Debug|Any CPU.ActiveCfg = Debug|Any CPU
     3871    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Modeling Debug|Any CPU.Build.0 = Debug|Any CPU
     3872    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Modeling Debug|x64.ActiveCfg = Debug|Any CPU
     3873    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Modeling Debug|x86.ActiveCfg = Debug|Any CPU
     3874    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Release|Any CPU.ActiveCfg = Release|Any CPU
     3875    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Release|Any CPU.Build.0 = Release|Any CPU
     3876    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Release|x64.ActiveCfg = Release|Any CPU
     3877    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Release|x86.ActiveCfg = Release|Any CPU
     3878    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.v3.2 Debug|Any CPU.ActiveCfg = Debug|Any CPU
     3879    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.v3.2 Debug|Any CPU.Build.0 = Debug|Any CPU
     3880    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.v3.2 Debug|x64.ActiveCfg = Debug|Any CPU
     3881    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.v3.2 Debug|x86.ActiveCfg = Debug|Any CPU
     3882    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Visualization Debug|Any CPU.ActiveCfg = Debug|Any CPU
     3883    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Visualization Debug|Any CPU.Build.0 = Debug|Any CPU
     3884    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Visualization Debug|x64.ActiveCfg = Debug|Any CPU
     3885    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB}.Visualization Debug|x86.ActiveCfg = Debug|Any CPU
    38533886  EndGlobalSection
    38543887  GlobalSection(SolutionProperties) = preSolution
    … …  
    39343967    {80F7FADA-549D-4151-8856-79B620A50DBA} = {410732DB-725A-4824-896B-C298978343C0}
    39353968    {ABAE70E0-14E3-4588-B6BB-15DF022985C3} = {410732DB-725A-4824-896B-C298978343C0}
     3969    {DFAC1BEA-6D9D-477F-AC7B-E64977644DDB} = {410732DB-725A-4824-896B-C298978343C0}
    39363970    {A9E282EA-180F-4233-B809-AEDF0787545C} = {78982D7C-D63D-4A3D-AE1F-F58AC007603B}
    39373971    {BF7D9494-A586-457B-8DF9-ED599F9E6A71} = {78982D7C-D63D-4A3D-AE1F-F58AC007603B}

  • trunk/sources/HeuristicLab/CopyAssemblies.cmd

    r2056 r2058  
    5050copy "%SolutionDir%\HeuristicLab.GP.StructureIdentification.TimeSeries\3.3\%Outdir%\HeuristicLab.GP.StructureIdentification.TimeSeries-3.3.dll" .\plugins
    5151copy "%SolutionDir%\HeuristicLab.Grid\3.2\%Outdir%\HeuristicLab.Grid-3.2.dll" .\plugins
     52copy "%SolutionDir%\HeuristicLab.Grid.HiveBridge\3.2\%Outdir%\HeuristicLab.Grid.HiveBridge-3.2.dll" .\plugins
    5253copy "%SolutionDir%\HeuristicLab.Hive.Client.Common\3.2\%Outdir%\HeuristicLab.Hive.Client.Common-3.2.dll" .\plugins
    5354copy "%SolutionDir%\HeuristicLab.Hive.Client.Communication\3.2\%Outdir%\HeuristicLab.Hive.Client.Communication-3.2.dll" .\plugins
Note: See TracChangeset for help on using the changeset viewer.