Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
02/21/11 17:35:42 (14 years ago)
Author:
cneumuel
Message:

#1233

  • fixed handling of StateLog in DataLayer
  • extended unit tests
  • changed style of service calls to OKB-like style (using delegates)
  • added possibility that parent jobs can be finished immediately when child jobs are finished
Location:
branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests/DaoTests.cs

    r5511 r5526  
    77
    88namespace HeuristicLab.Services.Hive.Tests {
     9  using System.Threading;
    910  using DT = HeuristicLab.Services.Hive.Common.DataTransfer;
    1011
    … …  
    1718      ServiceLocator.Instance = new MockServiceLocator(ServiceLocator.Instance);
    1819    }
    19    
     20
    2021    private IHiveService GetLocalService() {
    2122      return new HiveService();
    … …  
    2526    public void TestJobDao() {
    2627      IHiveDao dao = ServiceLocator.Instance.HiveDao;
    27      
     28
    2829      DT.Job job1 = new DT.Job();
    2930      job1.SetState(JobState.Offline, Guid.NewGuid());
    … …  
    6465        Assert.IsTrue(Math.Abs((job1.StateLog[i].DateTime - job1loaded.StateLog[i].DateTime).TotalSeconds) < 1);
    6566      }
     67     
     68      job1 = job1loaded;
     69
     70      // test jobstates
     71      job1.SetState(JobState.Transferring); Thread.Sleep(10);
     72      job1.SetState(JobState.Calculating); Thread.Sleep(10);
     73      job1.SetState(JobState.Transferring); Thread.Sleep(10);
     74      job1.SetState(JobState.Finished); Thread.Sleep(10);
     75      dao.UpdateJob(job1);
     76
     77      job1loaded = dao.GetJob(job1.Id);
     78      for (int i = 0; i < job1.StateLog.Count; i++) {
     79        Assert.AreEqual(job1.Id, job1loaded.StateLog[i].JobId);
     80        Assert.AreEqual(job1.StateLog[i].State, job1loaded.StateLog[i].State);
     81        Assert.AreEqual(job1.StateLog[i].SlaveId, job1loaded.StateLog[i].SlaveId);
     82        Assert.AreEqual(job1.StateLog[i].UserId, job1loaded.StateLog[i].UserId);
     83        Assert.AreEqual(job1.StateLog[i].Exception, job1loaded.StateLog[i].Exception);
     84        Assert.IsTrue(Math.Abs((job1.StateLog[i].DateTime - job1loaded.StateLog[i].DateTime).TotalSeconds) < 1);
     85      }
    6686
    6787      dao.DeleteJob(job1.Id);
    … …  
    175195      //Assert.AreEqual(he.LastAccessed, heLoaded.LastAccessed);
    176196      //Assert.AreEqual(he.DateCreated, heLoaded.DateCreated);
    177      
     197
    178198      DT.Job jobLoaded = dao.GetJob(he.RootJobId);
    179199      Assert.AreEqual(job.Id, jobLoaded.Id);

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests/HeuristicLab 3.3.exe.config

    r5095 r5526  
    2727    </roleManager>
    2828  </system.web>
     29 
    2930  <system.serviceModel>
    3031    <behaviors>

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests/HeuristicLab.Services.Hive.Tests-3.4.csproj

    r5402 r5526  
    333333    <Reference Include="System.Drawing" />
    334334    <Reference Include="System.Runtime.Serialization" />
     335    <Reference Include="System.Security" />
    335336    <Reference Include="System.ServiceModel" />
    336337    <Reference Include="System.ServiceProcess" />
     338    <Reference Include="System.Web" />
     339    <Reference Include="System.Web.ApplicationServices" />
    337340    <Reference Include="System.Windows.Forms" />
    338341    <Reference Include="System.Xml" />
    … …  
    397400    <None Include="HeuristicLab 3.3.exe.config">
    398401      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     402      <SubType>Designer</SubType>
    399403    </None>
    400404    <Shadow Include="Test References\HeuristicLab.Clients.Hive.Views-3.4.accessor" />

  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests/ServiceTests.cs

    r5511 r5526  
    2424using System.Linq;
    2525using System.Threading;
    26 using HeuristicLab.Clients.Hive;
    27 using HeuristicLab.Clients.Hive.Slave.Tests;
    28 using HeuristicLab.Hive;
    2926using HeuristicLab.Services.Hive.Common;
    3027using HeuristicLab.Services.Hive.Common.DataTransfer;
    … …  
    3431namespace HeuristicLab.Services.Hive.Tests {
    3532
    36   using System.Diagnostics;
    3733  using DT = HeuristicLab.Services.Hive.Common.DataTransfer;
    3834
    … …  
    7066      DT.JobData jobData = new DT.JobData() {
    7167        //Data = PersistenceUtil.Serialize(new MockJob(500, true))
    72         Data = new byte[10000000]
     68        Data = new byte[10000]
    7369      };
    7470
    … …  
    119115      Assert.AreEqual(JobState.Waiting, jobLoaded.State);
    120116      Assert.IsTrue(job.PluginsNeededIds.SequenceEqual(jobLoaded.PluginsNeededIds));
    121      
     117
    122118      DT.JobData jobDataLoaded = service.GetJobData(job.Id);
    123119      Assert.AreEqual(job.Id, jobDataLoaded.JobId);
    … …  
    132128
    133129      // test assigned ressources
    134       var actions = service.Heartbeat(new Heartbeat() { SlaveId = slave.Id, AssignJob = true, FreeCores = 4, FreeMemory = 1024, JobProgress = new Dictionary<Guid,TimeSpan>() });
     130      var actions = service.Heartbeat(new Heartbeat() { SlaveId = slave.Id, AssignJob = true, FreeCores = 4, FreeMemory = 1024, JobProgress = new Dictionary<Guid, TimeSpan>() });
    135131      Assert.AreEqual(1, actions.Count);
    136132      Assert.AreEqual(MessageContainer.MessageType.CalculateJob, actions[0].Message);
    … …  
    169165    List<DT.Job> jobs = new List<DT.Job>();
    170166
     167
    171168    [TestMethod]
    172     public void TestHeartbeats() {
     169    public void TestParentJobs() {
    173170      var service = GetLocalService();
    174       // check if group already exists and delete
    175       var existingSlaveGroup = service.GetSlaveGroups().SingleOrDefault(g => g.Name == "TestGroup");
    176       if (existingSlaveGroup != null) {
    177         var slavesToDelete = service.GetSlaves().Where(s => s.ParentResourceId == existingSlaveGroup.Id);
    178         foreach (var slave in slavesToDelete) service.DeleteSlave(slave.Id);
    179         service.DeleteSlaveGroup(existingSlaveGroup.Id);
    180       }
    181 
    182       Guid groupId = service.AddSlaveGroup(new SlaveGroup() { Name = "TestGroup", Description = "Used for unit tests" });
    183 
    184       // create slaves
    185       var slaves = new List<DT.Slave>();
    186       for (int i = 0; i < 1; i++) {
    187         DT.Slave slave = new DT.Slave() {
    188           Cores = 2,
    189           Memory = 4096,
    190           Name = "Slave " + i,
    191           IsAllowedToCalculate = true,
    192           SlaveState = SlaveState.Idle,
    193           CpuSpeed = 2800,
    194           FreeCores = 2,
    195           FreeMemory = 3000
    196         };
    197         // check if slave with this name already exists and delete
    198         var existingSlave = service.GetSlaves().Where(s => s.Name == slave.Name).SingleOrDefault();
    199         if (existingSlave != null) service.DeleteSlave(existingSlave.Id);
    200 
    201         slave.Id = service.AddSlave(slave);
    202         service.AddResourceToGroup(groupId, slave.Id);
    203         slaves.Add(slave);
    204       }
    205 
    206       // create jobs with different group, they should not be assigned
    207       existingSlaveGroup = service.GetSlaveGroups().SingleOrDefault(g => g.Name == "DummyGroup");
    208       if (existingSlaveGroup != null) service.DeleteSlaveGroup(existingSlaveGroup.Id);
    209 
    210       Guid dummyGroupId = service.AddSlaveGroup(new SlaveGroup() { Name = "DummyGroup", Description = "Used for unit tests; jobs from this group shall not be calculated" });
    211       // create dummy jobs
    212       var dummyJobs = new List<Job>();
    213       for (int i = 0; i < 2; i++) {
    214         Job job = new Job() {
    215           CoresNeeded = 1, MemoryNeeded = 0
    216         };
    217         JobData jobData = new JobData() { Data = PersistenceUtil.Serialize(new MockJob(500, false)) };
    218         job.Id = service.AddJob(job, jobData, new List<Guid> { dummyGroupId });
    219         dummyJobs.Add(job);
    220       }
    221 
    222       // create jobs
    223       for (int i = 0; i < 2; i++) {
    224         Job job = new Job() {
    225           CoresNeeded = 1, MemoryNeeded = 0
    226         };
    227         JobData jobData = new JobData() { Data = PersistenceUtil.Serialize(new MockJob(500, false)) };
    228         job.Id = service.AddJob(job, jobData, new List<Guid> { groupId });
    229         jobs.Add(job);
    230       }
    231 
    232       // send heartbeats
    233       foreach (var slave in slaves) {
    234         new Thread(new ParameterizedThreadStart(RunSlaveThread)).Start(slave);
    235       }
    236 
    237       IEnumerable<LightweightJob> lightweightJobs;
    238       do {
    239         Thread.Sleep(500);
    240         lightweightJobs = service.GetLightweightJobs(jobs.Select(x => x.Id));
    241       } while (!lightweightJobs.All(x => x.State == JobState.Finished));
    242 
    243       // delete slaves
    244       foreach (var slave in slaves) {
    245         service.DeleteSlave(slave.Id);
    246         Assert.AreEqual(null, service.GetSlave(slave.Id));
    247       }
    248 
    249       // delete groups
    250       service.DeleteSlaveGroup(groupId);
    251       service.DeleteSlaveGroup(dummyGroupId);
    252 
    253       // delete jobs
    254       foreach (var job in jobs) {
    255         service.DeleteJob(job.Id);
    256       }
    257 
    258       // delete dummy jobs
    259       foreach (var job in dummyJobs) {
    260         service.DeleteJob(job.Id);
    261       }
     171
     172      // create hive experiment
     173      DT.HiveExperiment experiment = new DT.HiveExperiment() {
     174        Name = "TestExperiment",
     175        Description = ""
     176      };
     177
     178      // create parent job
     179      DT.Job parentJob = new DT.Job() {
     180        CoresNeeded = 1,
     181        MemoryNeeded = 0,
     182        Priority = 0,
     183        IsParentJob = true,
     184        FinishWhenChildJobsFinished = true
     185      };
     186      parentJob.SetState(JobState.Offline);
     187
     188      DT.JobData parentJobData = new DT.JobData() { Data = new byte[0] };
     189
     190      // create child job
     191      DT.Job childJob = new DT.Job() {
     192        CoresNeeded = 1,
     193        MemoryNeeded = 0,
     194        Priority = 0
     195      };
     196      parentJob.SetState(JobState.Offline);
     197
     198      DT.JobData childJobData = new DT.JobData() { Data = new byte[1000] };
     199
     200      // create slave
     201      DT.Slave slave = new Slave();
     202      slave.Id = Guid.NewGuid();
     203      slave.Name = "TestSlave";
     204      slave.Memory = 1024;
     205      slave.Cores = 4;
     206      slave.CpuSpeed = 2800;
     207      slave.OperatingSystem = "Windows 3.11";
     208      slave.CpuArchitecture = CpuArchitecture.x64;
     209
     210      // add slave
     211      service.AddSlave(slave);
     212
     213      // add parent job
     214      parentJob.Id = service.AddJob(parentJob, parentJobData, new List<Guid> { slave.Id });
     215      experiment.RootJobId = parentJob.Id;
     216
     217      // add child job
     218      childJob.Id = service.AddChildJob(parentJob.Id, childJob, childJobData);
     219      childJob.ParentJobId = parentJob.Id;
     220
     221      // add hive experiment
     222      experiment.Id = service.AddHiveExperiment(experiment);
     223
     224      // test child job
     225      var childJobLoaded = service.GetJob(childJob.Id);
     226      Assert.AreEqual(childJob.ParentJobId, childJobLoaded.ParentJobId);
     227      Assert.AreEqual(JobState.Waiting, childJobLoaded.State);
     228      Assert.AreEqual(false, childJobLoaded.FinishWhenChildJobsFinished);
     229      Assert.AreEqual(false, childJobLoaded.IsParentJob);
     230
     231      // test parent job
     232      var parentJobLoaded = service.GetJob(parentJob.Id);
     233      Assert.AreEqual(JobState.Waiting, parentJobLoaded.State);
     234      Assert.AreEqual(true, parentJobLoaded.FinishWhenChildJobsFinished);
     235      Assert.AreEqual(true, parentJobLoaded.IsParentJob);
     236
     237      // test heartbeat
     238      var actions = service.Heartbeat(new Heartbeat() { SlaveId = slave.Id, AssignJob = true, FreeCores = 4, FreeMemory = 1024, JobProgress = new Dictionary<Guid, TimeSpan>() });
     239      Assert.AreEqual(1, actions.Count); // only the child job should be assigned
     240      Assert.AreEqual(MessageContainer.MessageType.CalculateJob, actions[0].Message);
     241      Assert.AreEqual(childJob.Id, actions[0].JobId);
     242
     243      // lifecycle - let it process one server-heartbeat; the parent job must NOT be set to finished
     244      var lifecycleManager = new LifecycleManager();
     245      lifecycleManager.Start();
     246      Thread.Sleep(1000);
     247      lifecycleManager.Stop();
     248
     249      parentJobLoaded = service.GetJob(parentJob.Id);
     250      Assert.AreEqual(JobState.Waiting, parentJobLoaded.State);
     251
     252      // set child job to finished
     253      childJobLoaded.SetState(JobState.Finished, slave.Id, "");
     254      service.UpdateJob(childJobLoaded);
     255
     256      // lifecycle - let it process one server-heartbeat; this should set the parent job to finished
     257      lifecycleManager.Start();
     258      Thread.Sleep(1000);
     259      lifecycleManager.Stop();
     260
     261      // test if parent job is finished
     262      parentJobLoaded = service.GetJob(parentJob.Id);
     263      Assert.AreEqual(JobState.Finished, parentJobLoaded.State);
     264
     265      // delete experiment
     266      service.DeleteHiveExperiment(experiment.Id);
     267      Assert.AreEqual(null, service.GetJob(parentJob.Id));
     268      Assert.AreEqual(null, service.GetJob(childJob.Id));
     269
     270      service.DeleteSlave(slave.Id);
    262271    }
    263272
    264     public void RunSlaveThread(object slaveobj) {
    265       try {
    266         var service = GetLocalService();
    267         Slave slave = (Slave)slaveobj;
    268         int freeCores = slave.Cores.Value;
    269 
    270         for (int i = 0; i < 10; i++) {
    271 
    272           var messages = service.Heartbeat(new Heartbeat() { SlaveId = slave.Id, FreeMemory = 2423, FreeCores = freeCores, JobProgress = new Dictionary<Guid, TimeSpan>() });
    273           if (messages.Count == 0) {
    274             Debug.WriteLine("No job available");
    275             return; // no more jobs
    276           }
    277 
    278           Debug.WriteLine("Messages: {0}", string.Join(", ", messages.Select(m => m.Message)));
    279 
    280           Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.AbortJob).Count() == 0);
    281           Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.SayHello).Count() == 0);
    282           Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.PauseJob).Count() == 0);
    283 
    284           var calculateJobMessage = messages.Where(x => x.Message == MessageContainer.MessageType.CalculateJob).SingleOrDefault();
    285           if (calculateJobMessage != null) {
    286             if (!jobs.Select(j => j.Id).Contains(calculateJobMessage.JobId))
    287               Assert.Fail("Got job which was not assigned to the slavegroup");
    288 
    289             Debug.WriteLine("Job available, calculating");
    290             Job job = service.GetJob(calculateJobMessage.JobId);
    291 
    292             JobData jobData = service.GetJobData(job.Id);
    293             IJob deserializedJob = PersistenceUtil.Deserialize<IJob>(jobData.Data);
    294             deserializedJob.Start();
    295             job.SetState(JobState.Finished);
    296             jobs.Where(x => x.Id == job.Id).Single().SetState(JobState.Finished);
    297             jobData.Data = PersistenceUtil.Serialize(deserializedJob);
    298             service.UpdateJobData(job, jobData);
    299             Debug.WriteLine("finished calculating");
    300           }
    301         }
    302       }
    303       catch (Exception e) {
    304         Assert.Fail(e.Message, e);
    305       }
    306     }
     273    //[TestMethod]
     274    //public void TestHeartbeats() {
     275    //  var service = GetLocalService();
     276    //  // check if group already exists and delete
     277    //  var existingSlaveGroup = service.GetSlaveGroups().SingleOrDefault(g => g.Name == "TestGroup");
     278    //  if (existingSlaveGroup != null) {
     279    //    var slavesToDelete = service.GetSlaves().Where(s => s.ParentResourceId == existingSlaveGroup.Id);
     280    //    foreach (var slave in slavesToDelete) service.DeleteSlave(slave.Id);
     281    //    service.DeleteSlaveGroup(existingSlaveGroup.Id);
     282    //  }
     283
     284    //  Guid groupId = service.AddSlaveGroup(new SlaveGroup() { Name = "TestGroup", Description = "Used for unit tests" });
     285
     286    //  // create slaves
     287    //  var slaves = new List<DT.Slave>();
     288    //  for (int i = 0; i < 1; i++) {
     289    //    DT.Slave slave = new DT.Slave() {
     290    //      Cores = 2,
     291    //      Memory = 4096,
     292    //      Name = "Slave " + i,
     293    //      IsAllowedToCalculate = true,
     294    //      SlaveState = SlaveState.Idle,
     295    //      CpuSpeed = 2800,
     296    //      FreeCores = 2,
     297    //      FreeMemory = 3000
     298    //    };
     299    //    // check if slave with this name already exists and delete
     300    //    var existingSlave = service.GetSlaves().Where(s => s.Name == slave.Name).SingleOrDefault();
     301    //    if (existingSlave != null) service.DeleteSlave(existingSlave.Id);
     302
     303    //    slave.Id = service.AddSlave(slave);
     304    //    service.AddResourceToGroup(groupId, slave.Id);
     305    //    slaves.Add(slave);
     306    //  }
     307
     308    //  // create jobs with different group, they should not be assigned
     309    //  existingSlaveGroup = service.GetSlaveGroups().SingleOrDefault(g => g.Name == "DummyGroup");
     310    //  if (existingSlaveGroup != null) service.DeleteSlaveGroup(existingSlaveGroup.Id);
     311
     312    //  Guid dummyGroupId = service.AddSlaveGroup(new SlaveGroup() { Name = "DummyGroup", Description = "Used for unit tests; jobs from this group shall not be calculated" });
     313    //  // create dummy jobs
     314    //  var dummyJobs = new List<Job>();
     315    //  for (int i = 0; i < 2; i++) {
     316    //    Job job = new Job() {
     317    //      CoresNeeded = 1, MemoryNeeded = 0
     318    //    };
     319    //    JobData jobData = new JobData() { Data = PersistenceUtil.Serialize(new MockJob(500, false)) };
     320    //    job.Id = service.AddJob(job, jobData, new List<Guid> { dummyGroupId });
     321    //    dummyJobs.Add(job);
     322    //  }
     323
     324    //  // create jobs
     325    //  for (int i = 0; i < 2; i++) {
     326    //    Job job = new Job() {
     327    //      CoresNeeded = 1, MemoryNeeded = 0
     328    //    };
     329    //    JobData jobData = new JobData() { Data = PersistenceUtil.Serialize(new MockJob(500, false)) };
     330    //    job.Id = service.AddJob(job, jobData, new List<Guid> { groupId });
     331    //    jobs.Add(job);
     332    //  }
     333
     334    //  // send heartbeats
     335    //  foreach (var slave in slaves) {
     336    //    new Thread(new ParameterizedThreadStart(RunSlaveThread)).Start(slave);
     337    //  }
     338
     339    //  IEnumerable<LightweightJob> lightweightJobs;
     340    //  do {
     341    //    Thread.Sleep(500);
     342    //    lightweightJobs = service.GetLightweightJobs(jobs.Select(x => x.Id));
     343    //  } while (!lightweightJobs.All(x => x.State == JobState.Finished));
     344
     345    //  // delete slaves
     346    //  foreach (var slave in slaves) {
     347    //    service.DeleteSlave(slave.Id);
     348    //    Assert.AreEqual(null, service.GetSlave(slave.Id));
     349    //  }
     350
     351    //  // delete groups
     352    //  service.DeleteSlaveGroup(groupId);
     353    //  service.DeleteSlaveGroup(dummyGroupId);
     354
     355    //  // delete jobs
     356    //  foreach (var job in jobs) {
     357    //    service.DeleteJob(job.Id);
     358    //  }
     359
     360    //  // delete dummy jobs
     361    //  foreach (var job in dummyJobs) {
     362    //    service.DeleteJob(job.Id);
     363    //  }
     364    //}
     365
     366    //public void RunSlaveThread(object slaveobj) {
     367    //  try {
     368    //    var service = GetLocalService();
     369    //    Slave slave = (Slave)slaveobj;
     370    //    int freeCores = slave.Cores.Value;
     371
     372    //    for (int i = 0; i < 10; i++) {
     373
     374    //      var messages = service.Heartbeat(new Heartbeat() { SlaveId = slave.Id, FreeMemory = 2423, FreeCores = freeCores, JobProgress = new Dictionary<Guid, TimeSpan>() });
     375    //      if (messages.Count == 0) {
     376    //        Debug.WriteLine("No job available");
     377    //        return; // no more jobs
     378    //      }
     379
     380    //      Debug.WriteLine("Messages: {0}", string.Join(", ", messages.Select(m => m.Message)));
     381
     382    //      Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.AbortJob).Count() == 0);
     383    //      Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.SayHello).Count() == 0);
     384    //      Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.PauseJob).Count() == 0);
     385
     386    //      var calculateJobMessage = messages.Where(x => x.Message == MessageContainer.MessageType.CalculateJob).SingleOrDefault();
     387    //      if (calculateJobMessage != null) {
     388    //        if (!jobs.Select(j => j.Id).Contains(calculateJobMessage.JobId))
     389    //          Assert.Fail("Got job which was not assigned to the slavegroup");
     390
     391    //        Debug.WriteLine("Job available, calculating");
     392    //        Job job = service.GetJob(calculateJobMessage.JobId);
     393
     394    //        JobData jobData = service.GetJobData(job.Id);
     395    //        IJob deserializedJob = PersistenceUtil.Deserialize<IJob>(jobData.Data);
     396    //        deserializedJob.Start();
     397    //        job.SetState(JobState.Finished);
     398    //        jobs.Where(x => x.Id == job.Id).Single().SetState(JobState.Finished);
     399    //        jobData.Data = PersistenceUtil.Serialize(deserializedJob);
     400    //        service.UpdateJobData(job, jobData);
     401    //        Debug.WriteLine("finished calculating");
     402    //      }
     403    //    }
     404    //  }
     405    //  catch (Exception e) {
     406    //    Assert.Fail(e.Message, e);
     407    //  }
     408    //}
    307409  }
    308410}
Note: See TracChangeset for help on using the changeset viewer.