Changeset 5179 for branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/HiveExperiment.cs

Timestamp:

12/29/10 00:52:06 (14 years ago)

Author:

cneumuel

Message:

#1260

• migrated to .NET 4.0
• moved state-information about heartbeat timestamps into DB to reduce IIS-recycling issues
• optimized memory usage of ExperimentManager when lots of large jobs are downloaded

File:

• branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/HiveExperiment.cs (modified) (4 diffs)

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.Hive.ExperimentManager/3.3/HiveExperiment.cs

@@ -534 +534 @@
         IsProgressing = true;
         int totalJobCount = 0;
-        int jobCount = 0;
         JobResultList allResults;
-        IDictionary<Guid, SerializedJob> allSerializedJobs = new Dictionary<Guid, SerializedJob>();
+
         progress.Status = "Connecting to Server...";
-        using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
-          // fetch all JobDto objects to create the full tree of tree of HiveJob objects
+        using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
           progress.Status = "Downloading list of jobs...";
           allResults = service.Obj.GetChildJobResults(rootJobId.Value, true, true).Obj;
           totalJobCount = allResults.Count;
-
-          // download them first
-          foreach (JobResult jobResult in allResults) {
-            jobCount++;
-            progress.Status = string.Format("Downloading {0} of {1} jobs...", jobCount, totalJobCount);
-            allSerializedJobs.Add(jobResult.Id, service.Obj.GetLastSerializedResult(jobResult.Id).Obj);
-            progress.ProgressValue = (double)jobCount / totalJobCount;
-          }
-        }
-
-        jobCount = 1;
-        progress.Status = string.Format("Deserializing {0} of {1} jobs... ({2} kb)", jobCount, totalJobCount, allSerializedJobs[this.rootJobId.Value].SerializedJobData.Count() / 1024);
-        this.HiveJob = new HiveJob(allSerializedJobs[this.rootJobId.Value], false);
-        allSerializedJobs.Remove(this.rootJobId.Value); // reduce memory footprint
-        progress.ProgressValue = (double)jobCount / totalJobCount;
+        }
+
+        HiveJobDownloader downloader = new HiveJobDownloader(allResults.Select(x => x.Id));
+        downloader.StartAsync();
+
+        while (!downloader.IsFinished) {
+          progress.ProgressValue = downloader.FinishedCount / (double)totalJobCount;
+          progress.Status = string.Format("Downloading/deserializing jobs... ({0}/{1} finished)", downloader.FinishedCount, totalJobCount);
+          Thread.Sleep(500);
+        }
+        IDictionary<Guid, HiveJob> allHiveJobs = downloader.Results;
+
+        this.HiveJob = allHiveJobs[this.rootJobId.Value];
 
         if (this.HiveJob.JobDto.DateFinished.HasValue) {
@@ -571 +567 @@
         }
 
-        // build child-job tree
-        LoadChildResults(this.HiveJob, allResults, allSerializedJobs, progress, totalJobCount, ref jobCount);
+        BuildHiveJobTree(this.HiveJob, allResults, allHiveJobs);
         StartResultPolling();
       }
@@ -583 +578 @@
     }
 
-    private void LoadChildResults(HiveJob parentHiveJob, JobResultList allResults, IDictionary<Guid, SerializedJob> allSerializedJobs, IProgress progress, int totalJobCount, ref int jobCount) {
+    private void BuildHiveJobTree(HiveJob parentHiveJob, JobResultList allResults, IDictionary<Guid, HiveJob> allHiveJobs) {
       IEnumerable<JobResult> childResults = from result in allResults
                                             where result.ParentJobId.HasValue && result.ParentJobId.Value == parentHiveJob.JobDto.Id
@@ -589 +584 @@
                                             select result;
       foreach (JobResult jobResult in childResults) {
-        jobCount++;
-        progress.Status = string.Format("Deserializing {0} of {1} jobs ({2} kb)...", jobCount, totalJobCount, allSerializedJobs[jobResult.Id].SerializedJobData.Count() / 1024);
-        OptimizerJob optimizerJob = null;
-        try {
-          optimizerJob = SerializedJob.Deserialize<OptimizerJob>(allSerializedJobs[jobResult.Id].SerializedJobData);
-        }
-        catch {
-          optimizerJob = null;
-        }
-        progress.ProgressValue = (double)jobCount / totalJobCount;
-        HiveJob childHiveJob = new HiveJob(optimizerJob, false);
+        HiveJob childHiveJob = allHiveJobs[jobResult.Id];
         parentHiveJob.AddChildHiveJob(childHiveJob);
-        childHiveJob.JobDto = allSerializedJobs[jobResult.Id].JobInfo;
-        allSerializedJobs.Remove(jobResult.Id); // reduce memory footprint
-        if (jobCount % 10 == 0) GC.Collect(); // this is needed or otherwise HL takes over the system when the number of jobs is high
-        LoadChildResults(childHiveJob, allResults, allSerializedJobs, progress, totalJobCount, ref jobCount);
+        BuildHiveJobTree(childHiveJob, allResults, allHiveJobs);
       }
     }