Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
02/22/11 13:54:31 (14 years ago)
Author:
cneumuel
Message:

#1233

  • merged changes in hive experiment manager from 3.3 (more reliable and efficient downloading of jobs)
File:
1 edited

Legend:

Unmodified
Added
Removed
  • branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs

    r5526 r5535  
    325325      JobData jobData;
    326326      List<IPluginDescription> plugins;
    327      
     327
    328328      if (hiveJob.OptimizerJob.ComputeInParallel &&
    329329        (hiveJob.OptimizerJob.Optimizer is Optimization.Experiment || hiveJob.OptimizerJob.Optimizer is Optimization.BatchRun)) {
     
    599599        IsProgressing = true;
    600600        int totalJobCount = 0;
    601         int jobCount = 0;
     601        IEnumerable<LightweightJob> allJobs;
     602
    602603        progress.Status = "Connecting to Server...";
    603         ServiceLocator.Instance.CallHiveService(service => {
    604           // fetch all Job objects to create the full tree of tree of HiveJob objects
    605           progress.Status = "Downloading list of jobs...";
    606           IEnumerable<LightweightJob> allResults = service.GetLightweightChildJobs(rootJobId, true, true);
    607           totalJobCount = allResults.Count();
    608 
    609           // download them first
    610           IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>();
    611           IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>();
    612           foreach (LightweightJob lightweightJob in allResults) {
    613             jobCount++;
    614             progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
    615             allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id));
    616             allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id));
    617             progress.ProgressValue = (double)jobCount / totalJobCount;
    618           }
    619 
    620           jobCount = 1;
    621           progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024);
    622           this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false);
    623           allJobDatas.Remove(this.rootJobId); // reduce memory footprint
    624           allJobs.Remove(this.rootJobId);
    625           progress.ProgressValue = (double)jobCount / totalJobCount;
    626 
    627           if (this.HiveJob.Job.DateFinished.HasValue) {
    628             this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated;
    629             this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value;
    630             this.ExecutionState = Core.ExecutionState.Stopped;
    631             OnStopped();
    632           } else {
    633             this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated;
    634             this.lastUpdateTime = DateTime.Now;
    635             this.ExecutionState = Core.ExecutionState.Started;
    636             OnStarted();
    637           }
    638 
    639           // build child-job tree
    640           LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
    641           StartResultPolling();
    642         });
     604        // fetch all Job objects to create the full tree of tree of HiveJob objects
     605        progress.Status = "Downloading list of jobs...";
     606        allJobs = ServiceLocator.Instance.CallHiveService(s => s.GetLightweightChildJobs(rootJobId, true, true));
     607        totalJobCount = allJobs.Count();
     608
     609        HiveJobDownloader downloader = new HiveJobDownloader(allJobs.Select(x => x.Id));
     610        downloader.StartAsync();
     611
     612        while (!downloader.IsFinished) {
     613          progress.ProgressValue = downloader.FinishedCount / (double)totalJobCount;
     614          progress.Status = string.Format("Downloading/deserializing jobs... ({0}/{1} finished)", downloader.FinishedCount, totalJobCount);
     615          Thread.Sleep(500);
     616        }
     617        IDictionary<Guid, HiveJob> allHiveJobs = downloader.Results;
     618
     619        this.HiveJob = allHiveJobs[this.rootJobId];
     620
     621
     622        //// download them first
     623        //IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>();
     624        //IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>();
     625        //foreach (LightweightJob lightweightJob in allResults) {
     626        //  jobCount++;
     627        //  progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
     628        //  allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id));
     629        //  allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id));
     630        //  progress.ProgressValue = (double)jobCount / totalJobCount;
     631        //}
     632
     633        //jobCount = 1;
     634        //progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024);
     635        //this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false);
     636        //allJobDatas.Remove(this.rootJobId); // reduce memory footprint
     637        //allJobs.Remove(this.rootJobId);
     638        //progress.ProgressValue = (double)jobCount / totalJobCount;
     639
     640        if (this.HiveJob.Job.DateFinished.HasValue) {
     641          this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated;
     642          this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value;
     643          this.ExecutionState = Core.ExecutionState.Stopped;
     644          OnStopped();
     645        } else {
     646          this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated;
     647          this.lastUpdateTime = DateTime.Now;
     648          this.ExecutionState = Core.ExecutionState.Started;
     649          OnStarted();
     650        }
     651
     652        // build child-job tree
     653        //LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
     654        BuildHiveJobTree(this.HiveJob, allJobs, allHiveJobs);
     655        StartResultPolling();
    643656      }
    644657      catch (Exception e) {
     
    650663    }
    651664
    652     private void LoadChildResults(IHiveService service, HiveJob parentHiveJob, IEnumerable<LightweightJob> allResults, IDictionary<Guid, Job> allJobs, IDictionary<Guid, JobData> allJobDatas, IProgress progress, int totalJobCount, ref int jobCount) {
    653       IEnumerable<LightweightJob> childResults = from result in allResults
    654                                                  where result.ParentJobId.HasValue && result.ParentJobId.Value == parentHiveJob.Job.Id
    655                                                  orderby result.DateCreated ascending
    656                                                  select result;
    657       foreach (LightweightJob lightweightJob in childResults) {
    658         jobCount++;
    659         progress.Status = string.Format("Deserializing {0} of {1} jobs ({2} kb)...", jobCount, totalJobCount, allJobDatas[lightweightJob.Id].Data.Count() / 1024);
    660         OptimizerJob optimizerJob = null;
    661         try {
    662           optimizerJob = PersistenceUtil.Deserialize<OptimizerJob>(allJobDatas[lightweightJob.Id].Data);
    663         }
    664         catch {
    665           optimizerJob = null;
    666         }
    667         progress.ProgressValue = (double)jobCount / totalJobCount;
    668         HiveJob childHiveJob = new HiveJob(optimizerJob, false);
     665    private void BuildHiveJobTree(HiveJob parentHiveJob, IEnumerable<LightweightJob> allJobs, IDictionary<Guid, HiveJob> allHiveJobs) {
     666      IEnumerable<LightweightJob> childJobs = from job in allJobs
     667                                              where job.ParentJobId.HasValue && job.ParentJobId.Value == parentHiveJob.Job.Id
     668                                              orderby job.DateCreated ascending
     669                                              select job;
     670      foreach (LightweightJob job in childJobs) {
     671        HiveJob childHiveJob = allHiveJobs[job.Id];
    669672        parentHiveJob.AddChildHiveJob(childHiveJob);
    670         childHiveJob.Job = allJobs[lightweightJob.Id];
    671         allJobDatas.Remove(lightweightJob.Id); // reduce memory footprint
    672         allJobs.Remove(lightweightJob.Id);
    673         if (jobCount % 10 == 0) GC.Collect(); // this is needed or otherwise HL takes over the system when the number of jobs is high
    674         LoadChildResults(service, childHiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
    675       }
    676     }
    677 
     673        BuildHiveJobTree(childHiveJob, allJobs, allHiveJobs);
     674      }
     675    }
     676   
    678677    private OptimizerJob LoadOptimizerJob(Guid jobId) {
    679       return ServiceLocator.Instance.CallHiveService(service => {
    680         JobData jobData = service.GetJobData(jobId);
    681         try {
    682           return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data);
    683         }
    684         catch {
    685           return null;
    686         }
    687       });
     678      JobData jobData = ServiceLocator.Instance.CallHiveService(s => s.GetJobData(jobId));
     679      try {
     680        return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data);
     681      }
     682      catch {
     683        return null;
     684      }
    688685    }
    689686    #endregion
Note: See TracChangeset for help on using the changeset viewer.