Changeset 5535 for branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs

Timestamp:

02/22/11 13:54:31 (14 years ago)

Author:

cneumuel

Message:

#1233

• merged changes in hive experiment manager from 3.3 (more reliable and efficient downloading of jobs)

File:

• branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs (modified) (3 diffs)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs

@@ -325 +325 @@
       JobData jobData;
       List<IPluginDescription> plugins;
-      
+
       if (hiveJob.OptimizerJob.ComputeInParallel &&
         (hiveJob.OptimizerJob.Optimizer is Optimization.Experiment || hiveJob.OptimizerJob.Optimizer is Optimization.BatchRun)) {
@@ -599 +599 @@
         IsProgressing = true;
         int totalJobCount = 0;
-        int jobCount = 0;
+        IEnumerable<LightweightJob> allJobs;
+
         progress.Status = "Connecting to Server...";
-        ServiceLocator.Instance.CallHiveService(service => {
-          // fetch all Job objects to create the full tree of tree of HiveJob objects
-          progress.Status = "Downloading list of jobs...";
-          IEnumerable<LightweightJob> allResults = service.GetLightweightChildJobs(rootJobId, true, true);
-          totalJobCount = allResults.Count();
-
-          // download them first
-          IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>();
-          IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>();
-          foreach (LightweightJob lightweightJob in allResults) {
-            jobCount++;
-            progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
-            allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id));
-            allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id));
-            progress.ProgressValue = (double)jobCount / totalJobCount;
-          }
-
-          jobCount = 1;
-          progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024);
-          this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false);
-          allJobDatas.Remove(this.rootJobId); // reduce memory footprint
-          allJobs.Remove(this.rootJobId);
-          progress.ProgressValue = (double)jobCount / totalJobCount;
-
-          if (this.HiveJob.Job.DateFinished.HasValue) {
-            this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated;
-            this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value;
-            this.ExecutionState = Core.ExecutionState.Stopped;
-            OnStopped();
-          } else {
-            this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated;
-            this.lastUpdateTime = DateTime.Now;
-            this.ExecutionState = Core.ExecutionState.Started;
-            OnStarted();
-          }
-
-          // build child-job tree
-          LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
-          StartResultPolling();
-        });
+        // fetch all Job objects to create the full tree of tree of HiveJob objects
+        progress.Status = "Downloading list of jobs...";
+        allJobs = ServiceLocator.Instance.CallHiveService(s => s.GetLightweightChildJobs(rootJobId, true, true));
+        totalJobCount = allJobs.Count();
+
+        HiveJobDownloader downloader = new HiveJobDownloader(allJobs.Select(x => x.Id));
+        downloader.StartAsync();
+
+        while (!downloader.IsFinished) {
+          progress.ProgressValue = downloader.FinishedCount / (double)totalJobCount;
+          progress.Status = string.Format("Downloading/deserializing jobs... ({0}/{1} finished)", downloader.FinishedCount, totalJobCount);
+          Thread.Sleep(500);
+        }
+        IDictionary<Guid, HiveJob> allHiveJobs = downloader.Results;
+
+        this.HiveJob = allHiveJobs[this.rootJobId];
+
+
+        //// download them first
+        //IDictionary<Guid, Job> allJobs = new Dictionary<Guid, Job>();
+        //IDictionary<Guid, JobData> allJobDatas = new Dictionary<Guid, JobData>();
+        //foreach (LightweightJob lightweightJob in allResults) {
+        //  jobCount++;
+        //  progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
+        //  allJobs.Add(lightweightJob.Id, service.GetJob(lightweightJob.Id));
+        //  allJobDatas.Add(lightweightJob.Id, service.GetJobData(lightweightJob.Id));
+        //  progress.ProgressValue = (double)jobCount / totalJobCount;
+        //}
+
+        //jobCount = 1;
+        //progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allJobDatas[this.rootJobId].Data.Count() / 1024);
+        //this.HiveJob = new HiveJob(allJobs[this.rootJobId], allJobDatas[this.rootJobId], false);
+        //allJobDatas.Remove(this.rootJobId); // reduce memory footprint
+        //allJobs.Remove(this.rootJobId);
+        //progress.ProgressValue = (double)jobCount / totalJobCount;
+
+        if (this.HiveJob.Job.DateFinished.HasValue) {
+          this.ExecutionTime = this.HiveJob.Job.DateFinished.Value - this.HiveJob.Job.DateCreated;
+          this.lastUpdateTime = this.HiveJob.Job.DateFinished.Value;
+          this.ExecutionState = Core.ExecutionState.Stopped;
+          OnStopped();
+        } else {
+          this.ExecutionTime = DateTime.Now - this.HiveJob.Job.DateCreated;
+          this.lastUpdateTime = DateTime.Now;
+          this.ExecutionState = Core.ExecutionState.Started;
+          OnStarted();
+        }
+
+        // build child-job tree
+        //LoadChildResults(service, this.HiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
+        BuildHiveJobTree(this.HiveJob, allJobs, allHiveJobs);
+        StartResultPolling();
       }
       catch (Exception e) {
@@ -650 +663 @@
     }
 
-    private void LoadChildResults(IHiveService service, HiveJob parentHiveJob, IEnumerable<LightweightJob> allResults, IDictionary<Guid, Job> allJobs, IDictionary<Guid, JobData> allJobDatas, IProgress progress, int totalJobCount, ref int jobCount) {
-      IEnumerable<LightweightJob> childResults = from result in allResults
-                                                 where result.ParentJobId.HasValue && result.ParentJobId.Value == parentHiveJob.Job.Id
-                                                 orderby result.DateCreated ascending
-                                                 select result;
-      foreach (LightweightJob lightweightJob in childResults) {
-        jobCount++;
-        progress.Status = string.Format("Deserializing {0} of {1} jobs ({2} kb)...", jobCount, totalJobCount, allJobDatas[lightweightJob.Id].Data.Count() / 1024);
-        OptimizerJob optimizerJob = null;
-        try {
-          optimizerJob = PersistenceUtil.Deserialize<OptimizerJob>(allJobDatas[lightweightJob.Id].Data);
-        }
-        catch {
-          optimizerJob = null;
-        }
-        progress.ProgressValue = (double)jobCount / totalJobCount;
-        HiveJob childHiveJob = new HiveJob(optimizerJob, false);
+    private void BuildHiveJobTree(HiveJob parentHiveJob, IEnumerable<LightweightJob> allJobs, IDictionary<Guid, HiveJob> allHiveJobs) {
+      IEnumerable<LightweightJob> childJobs = from job in allJobs
+                                              where job.ParentJobId.HasValue && job.ParentJobId.Value == parentHiveJob.Job.Id
+                                              orderby job.DateCreated ascending
+                                              select job;
+      foreach (LightweightJob job in childJobs) {
+        HiveJob childHiveJob = allHiveJobs[job.Id];
         parentHiveJob.AddChildHiveJob(childHiveJob);
-        childHiveJob.Job = allJobs[lightweightJob.Id];
-        allJobDatas.Remove(lightweightJob.Id); // reduce memory footprint
-        allJobs.Remove(lightweightJob.Id);
-        if (jobCount % 10 == 0) GC.Collect(); // this is needed or otherwise HL takes over the system when the number of jobs is high
-        LoadChildResults(service, childHiveJob, allResults, allJobs, allJobDatas, progress, totalJobCount, ref jobCount);
-      }
-    }
-
+        BuildHiveJobTree(childHiveJob, allJobs, allHiveJobs);
+      }
+    }
+    
     private OptimizerJob LoadOptimizerJob(Guid jobId) {
-      return ServiceLocator.Instance.CallHiveService(service => {
-        JobData jobData = service.GetJobData(jobId);
-        try {
-          return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data);
-        }
-        catch {
-          return null;
-        }
-      });
+      JobData jobData = ServiceLocator.Instance.CallHiveService(s => s.GetJobData(jobId));
+      try {
+        return PersistenceUtil.Deserialize<OptimizerJob>(jobData.Data);
+      }
+      catch {
+        return null;
+      }
     }
     #endregion