- Timestamp:
- 02/22/11 13:54:31 (14 years ago)
- Location:
- branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4
- Files:
-
- 1 added
- 3 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs
r5526 r5535 325 325 JobData jobData; 326 326 List<IPluginDescription> plugins; 327 327 328 328 if (hiveJob.OptimizerJob.ComputeInParallel && 329 329 (hiveJob.OptimizerJob.Optimizer is Optimization.Experiment || hiveJob.OptimizerJob.Optimizer is Optimization.BatchRun)) { … … 599 599 IsProgressing = true; 600 600 int totalJobCount = 0; 601 int jobCount = 0; 601 IEnumerable<LightweightJob> allJobs; 602 602 603 progress.Status = "Connecting to Server..."; 603 ServiceLocator.Instance.CallHiveService(service => { 604 // fetch all Job objects to create the full tree of tree of HiveJob objects 605 progress.Status = "Downloading list of jobs..."; 606 IEnumerable<LightweightJob> allResults = service.GetLightweightChildJobs(rootJobId, true, true); 607 totalJobCount = allResults.Count(); 608 609 // download them first 610 IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>(); 611 IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>(); 612 foreach (LightweightJob lightweightJob in allResults) { 613 jobCount++; 614 progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount); 615 allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id)); 616 allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id)); 617 progress.ProgressValue = (double)jobCount / totalJobCount; 618 } 619 620 jobCount = 1; 621 progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024); 622 this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false); 623 allJobDatas.Remove(this.rootJobId); // reduce memory footprint 624 allJobs.Remove(this.rootJobId); 625 progress.ProgressValue = (double)jobCount / totalJobCount; 626 627 if (this.HiveJob.Job.DateFinished.HasValue) { 628 this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated; 629 this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value; 630 this.ExecutionState = Core.ExecutionState.Stopped; 631 OnStopped(); 632 } else { 633 this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated; 634 this.lastUpdateTime = DateTime.Now; 635 this.ExecutionState = Core.ExecutionState.Started; 636 OnStarted(); 637 } 638 639 // build child-job tree 640 LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount); 641 StartResultPolling(); 642 }); 604 // fetch all Job objects to create the full tree of tree of HiveJob objects 605 progress.Status = "Downloading list of jobs..."; 606 allJobs = ServiceLocator.Instance.CallHiveService(s => s.GetLightweightChildJobs(rootJobId, true, true)); 607 totalJobCount = allJobs.Count(); 608 609 HiveJobDownloader downloader = new HiveJobDownloader(allJobs.Select(x => x.Id)); 610 downloader.StartAsync(); 611 612 while (!downloader.IsFinished) { 613 progress.ProgressValue = downloader.FinishedCount / (double)totalJobCount; 614 progress.Status = string.Format("Downloading/deserializing jobs... ({0}/{1} finished)", downloader.FinishedCount, totalJobCount); 615 Thread.Sleep(500); 616 } 617 IDictionary<Guid, HiveJob> allHiveJobs = downloader.Results; 618 619 this.HiveJob = allHiveJobs[this.rootJobId]; 620 621 622 //// download them first 623 //IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>(); 624 //IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>(); 625 //foreach (LightweightJob lightweightJob in allResults) { 626 // jobCount++; 627 // progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount); 628 // allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id)); 629 // allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id)); 630 // progress.ProgressValue = (double)jobCount / totalJobCount; 631 //} 632 633 //jobCount = 1; 634 //progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024); 635 //this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false); 636 //allJobDatas.Remove(this.rootJobId); // reduce memory footprint 637 //allJobs.Remove(this.rootJobId); 638 //progress.ProgressValue = (double)jobCount / totalJobCount; 639 640 if (this.HiveJob.Job.DateFinished.HasValue) { 641 this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated; 642 this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value; 643 this.ExecutionState = Core.ExecutionState.Stopped; 644 OnStopped(); 645 } else { 646 this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated; 647 this.lastUpdateTime = DateTime.Now; 648 this.ExecutionState = Core.ExecutionState.Started; 649 OnStarted(); 650 } 651 652 // build child-job tree 653 //LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount); 654 BuildHiveJobTree(this.HiveJob, allJobs, allHiveJobs); 655 StartResultPolling(); 643 656 } 644 657 catch (Exception e) { … … 650 663 } 651 664 652 private void LoadChildResults(IHiveService service, HiveJob parentHiveJob, IEnumerable<LightweightJob> allResults, IDictionary<Guid, Job> allJobs, IDictionary<Guid, JobData> allJobDatas, IProgress progress, int totalJobCount, ref int jobCount) { 653 IEnumerable<LightweightJob> childResults = from result in allResults 654 where result.ParentJobId.HasValue && result.ParentJobId.Value == parentHiveJob.Job.Id 655 orderby result.DateCreated ascending 656 select result; 657 foreach (LightweightJob lightweightJob in childResults) { 658 jobCount++; 659 progress.Status = string.Format("Deserializing {0} of {1} jobs ({2} kb)...", jobCount, totalJobCount, allJobDatas[lightweightJob.Id].Data.Count() / 1024); 660 OptimizerJob optimizerJob = null; 661 try { 662 optimizerJob = PersistenceUtil.Deserialize<OptimizerJob>(allJobDatas[lightweightJob.Id].Data); 663 } 664 catch { 665 optimizerJob = null; 666 } 667 progress.ProgressValue = (double)jobCount / totalJobCount; 668 HiveJob childHiveJob = new HiveJob(optimizerJob, false); 665 private void BuildHiveJobTree(HiveJob parentHiveJob, IEnumerable<LightweightJob> allJobs, IDictionary<Guid, HiveJob> allHiveJobs) { 666 IEnumerable<LightweightJob> childJobs = from job in allJobs 667 where job.ParentJobId.HasValue && job.ParentJobId.Value == parentHiveJob.Job.Id 668 orderby job.DateCreated ascending 669 select job; 670 foreach (LightweightJob job in childJobs) { 671 HiveJob childHiveJob = allHiveJobs[job.Id]; 669 672 parentHiveJob.AddChildHiveJob(childHiveJob); 670 childHiveJob.Job = allJobs[lightweightJob.Id]; 671 allJobDatas.Remove(lightweightJob.Id); // reduce memory footprint 672 allJobs.Remove(lightweightJob.Id); 673 if (jobCount % 10 == 0) GC.Collect(); // this is needed or otherwise HL takes over the system when the number of jobs is high 674 LoadChildResults(service, childHiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount); 675 } 676 } 677 673 BuildHiveJobTree(childHiveJob, allJobs, allHiveJobs); 674 } 675 } 676 678 677 private OptimizerJob LoadOptimizerJob(Guid jobId) { 679 return ServiceLocator.Instance.CallHiveService(service => { 680 JobData jobData = service.GetJobData(jobId); 681 try { 682 return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data); 683 } 684 catch { 685 return null; 686 } 687 }); 678 JobData jobData = ServiceLocator.Instance.CallHiveService(s => s.GetJobData(jobId)); 679 try { 680 return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data); 681 } 682 catch { 683 return null; 684 } 688 685 } 689 686 #endregion -
branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobClient.cs
r5526 r5535 114 114 } 115 115 116 //public HiveJob(Job jobDto)117 // : this() {118 // this.Job = jobDto;119 //}120 121 //public HiveJob(LightweightJob lightweightJob)122 // : this() {123 // UpdateFromLightweightJob(lightweightJob);124 //}125 126 116 public HiveJob(OptimizerJob optimizerJob, bool autoCreateChildHiveJobs) 127 117 : this() { … … 145 135 this.OptimizerJob = null; 146 136 } 137 this.childHiveJobs = new ItemList<HiveJob>(); 147 138 this.syncJobsWithOptimizers = true; 148 139 } -
branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/HeuristicLab.Clients.Hive-3.4.csproj
r5458 r5535 118 118 <Compile Include="Exceptions\JobResultPollingException.cs" /> 119 119 <Compile Include="Exceptions\OptimizerNotFoundException.cs" /> 120 <Compile Include="ExperimentManager\HiveJobDownloader.cs" /> 120 121 <Compile Include="ExperimentManager\PluginClient.cs" /> 121 122 <Compile Include="Exceptions\ResourceNotFoundException.cs" />
Note: See TracChangeset
for help on using the changeset viewer.