Changeset 5268 for branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.HiveEngine/3.3

Timestamp:

01/10/11 15:48:54 (14 years ago)

Author:

cneumuel

Message:

#1247

• optimized memory consumption
• increased robustness

File:

• branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.HiveEngine/3.3/HiveEngine.cs (modified) (7 diffs)

branches/HeuristicLab.Hive-3.3/sources/HeuristicLab.Hive/HeuristicLab.HiveEngine/3.3/HiveEngine.cs

@@ -67 +67 @@
           coll = (OperationCollection)next;
           if (coll.Parallel) {
+            // clone the parent scope here and reuse it for each operation. otherwise for each job the whole scope-tree first needs to be copied and then cleaned, which causes a lot of work for the Garbage Collector
+            IScope parentScopeClone = (IScope)((IAtomicOperation)coll.First()).Scope.Parent.Clone();
+            parentScopeClone.SubScopes.Clear();
+            parentScopeClone.ClearParentScopes();
+
             OperationJob[] jobs = new OperationJob[coll.Count];
             for (int i = 0; i < coll.Count; i++) {
@@ -72 +77 @@
             }
 
-            IScope[] scopes = ExecuteOnHive(jobs, cancellationToken);
+            IScope[] scopes = ExecuteOnHive(jobs, parentScopeClone, cancellationToken);
 
             for (int i = 0; i < coll.Count; i++) {
@@ -136 +141 @@
     /// </summary>
     /// <param name="jobs"></param>
-    private IScope[] ExecuteOnHive(OperationJob[] jobs, CancellationToken cancellationToken) {
+    private IScope[] ExecuteOnHive(OperationJob[] jobs, IScope parentScopeClone, CancellationToken cancellationToken) {
       LogMessage(string.Format("Executing {0} operations on the hive.", jobs.Length));
       IScope[] scopes = new Scope[jobs.Length];
@@ -160 +165 @@
 
           uploadTasks.Add(Task.Factory.StartNew<JobDto>((keyValuePairObj) => {
-            return UploadJob(pluginsNeeded, keyValuePairObj, cancellationToken);
+            return UploadJob(pluginsNeeded, keyValuePairObj, parentScopeClone, cancellationToken);
           }, new KeyValuePair<int, OperationJob>(i, job), cancellationToken));
         }
@@ -198 +203 @@
 
           Thread.Sleep(10000);
-          using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
-            results = service.Obj.GetJobResults(remainingJobIds).Obj;
-          }
-          var jobsFinished = results.Where(j => j.State == JobState.Finished || j.State == JobState.Failed || j.State == JobState.Aborted);
-          finishedCount += jobsFinished.Count();
-          var totalExecutionTime = TimeSpan.FromMilliseconds(results.Select(j => j.ExecutionTime).Union(executionTimes).Select(e => e.TotalMilliseconds).Sum());
-          LogMessage(string.Format("Results polled. Jobs finished: {0}/{1}, TotalExecutionTime: {2}", finishedCount, jobs.Length, totalExecutionTime));
-          foreach (var result in jobsFinished) {
-            if (result.State == JobState.Finished) {
-              downloadTasks.Add(Task.Factory.StartNew<OperationJob>((jobIdObj) => {
-                return DownloadJob(jobIndices, jobIdObj, cancellationToken);
-              }, result.Id, cancellationToken));
-            } else if (result.State == JobState.Aborted) {
-              LogMessage(string.Format("Job #{0} aborted (id: {1})", jobIndices[result.Id] + 1, result.Id));
-            } else if (result.State == JobState.Failed) {
-              LogMessage(string.Format("Job {0} failed (id: {1}): {2}", jobIndices[result.Id] + 1, result.Id, result.Exception));
-            }
-            remainingJobIds.Remove(result.Id);
-            executionTimes.Add(result.ExecutionTime);
+          try {
+            using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
+              results = service.Obj.GetJobResults(remainingJobIds).Obj;
+            }
+            var jobsFinished = results.Where(j => j.State == JobState.Finished || j.State == JobState.Failed || j.State == JobState.Aborted);
+            finishedCount += jobsFinished.Count();
+            var totalExecutionTime = TimeSpan.FromMilliseconds(results.Select(j => j.ExecutionTime).Union(executionTimes).Select(e => e.TotalMilliseconds).Sum());
+            LogMessage(string.Format("Results polled. Jobs finished: {0}/{1}, TotalExecutionTime: {2}", finishedCount, jobs.Length, totalExecutionTime));
+            foreach (var result in jobsFinished) {
+              if (result.State == JobState.Finished) {
+                downloadTasks.Add(Task.Factory.StartNew<OperationJob>((jobIdObj) => {
+                  return DownloadJob(jobIndices, jobIdObj, cancellationToken);
+                }, result.Id, cancellationToken));
+              } else if (result.State == JobState.Aborted) {
+                LogMessage(string.Format("Job #{0} aborted (id: {1})", jobIndices[result.Id] + 1, result.Id));
+              } else if (result.State == JobState.Failed) {
+                LogMessage(string.Format("Job {0} failed (id: {1}): {2}", jobIndices[result.Id] + 1, result.Id, result.Exception));
+              }
+              remainingJobIds.Remove(result.Id);
+              executionTimes.Add(result.ExecutionTime);
+            }
+          }
+          catch (Exception e) {
+            LogException(e);
           }
         }
@@ -254 +264 @@
 
     private static object locker = new object();
-    private JobDto UploadJob(List<HivePluginInfoDto> pluginsNeeded, object keyValuePairObj, CancellationToken cancellationToken) {
+    private JobDto UploadJob(List<HivePluginInfoDto> pluginsNeeded, object keyValuePairObj, IScope parentScopeClone, CancellationToken cancellationToken) {
       var keyValuePair = (KeyValuePair<int, OperationJob>)keyValuePairObj;
       var groups = ResourceIds.Split(';');
@@ -272 +282 @@
       // clone operation and remove unnecessary scopes; don't do this earlier to avoid memory problems
       lock (locker) {
+        ((IAtomicOperation)keyValuePair.Value.Operation).Scope.Parent = parentScopeClone;
         keyValuePair.Value.Operation = (IOperation)keyValuePair.Value.Operation.Clone();
         if (keyValuePair.Value.Operation is IAtomicOperation)