Changeset 268 for trunk

Timestamp:

05/19/08 16:29:44 (17 years ago)

Author:

gkronber

Message:

improved RAM footprint of distributed-engine by serializing only the part of the scope-tree that contains the scope (including sub-scopes) on which a parallel operation 'operates' on. (ticket #153)

Location:

trunk/sources

Files:

• HeuristicLab.DistributedEngine/DistributedEngine.cs (modified) (3 diffs)
• HeuristicLab.DistributedEngine/JobManager.cs (modified) (1 diff)
• HeuristicLab.Grid/ProcessingEngine.cs (modified) (1 diff)

trunk/sources/HeuristicLab.DistributedEngine/DistributedEngine.cs

@@ -32 +32 @@
 using HeuristicLab.PluginInfrastructure;
 using System.Windows.Forms;
+using System.Diagnostics;
 
 namespace HeuristicLab.DistributedEngine {
@@ -92 +93 @@
             WaitHandle[] waithandles = new WaitHandle[compositeOperation.Operations.Count];
             int i = 0;
+            // HACK: assume that all atomicOperations have the same parent scope.
+            // 1) find that parent scope
+            // 2) remove all branches starting from the global scope that don't lead to the parentScope of the parallel operation
+            // 3) keep the branches to 'repair' the scope-tree later
+            // 4) for each parallel job attach only the sub-scope that this operation uses
+            // 5) after starting all parallel jobs restore the whole scope-tree
+            IScope parentScope = FindParentScope(GlobalScope, compositeOperation);
+            List<IList<IScope>> prunedScopes = new List<IList<IScope>>();
+            PruneToParentScope(GlobalScope, parentScope, prunedScopes);
+            List<IScope> subScopes = new List<IScope>(parentScope.SubScopes);
+            foreach(IScope scope in subScopes) {
+              parentScope.RemoveSubScope(scope);
+            }
             // start all parallel jobs
             foreach(AtomicOperation parOperation in compositeOperation.Operations) {
-              waithandles[i++] = jobManager.BeginExecuteOperation(OperatorGraph, GlobalScope, parOperation);
-            }
+              parentScope.AddSubScope(parOperation.Scope);
+              waithandles[i++] = jobManager.BeginExecuteOperation(GlobalScope, parOperation);
+              parentScope.RemoveSubScope(parOperation.Scope);
+            }
+            foreach(IScope scope in subScopes) {
+              parentScope.AddSubScope(scope);
+            }
+            prunedScopes.Reverse();
+            RestoreFullTree(GlobalScope, prunedScopes);
 
             // wait until all jobs are finished
@@ -124 +145 @@
     }
 
+    private void RestoreFullTree(IScope currentScope, IList<IList<IScope>> savedScopes) {
+      if(savedScopes.Count == 0) return;
+      IScope remainingBranch = currentScope.SubScopes[0];
+      currentScope.RemoveSubScope(remainingBranch);
+      IList<IScope> savedScopesForCurrent = savedScopes[0];
+      foreach(IScope savedScope in savedScopesForCurrent) {
+        currentScope.AddSubScope(savedScope);
+      }
+      savedScopes.RemoveAt(0);
+      RestoreFullTree(remainingBranch, savedScopes);
+    }
+
+    private IScope PruneToParentScope(IScope currentScope, IScope scope, IList<IList<IScope>> prunedScopes) {
+      if(currentScope == scope) return currentScope;
+      if(currentScope.SubScopes.Count == 0) return null;
+      IScope foundScope = null;
+      // try to find the searched scope in all my sub-scopes
+      foreach(IScope subScope in currentScope.SubScopes) {
+        foundScope = PruneToParentScope(subScope, scope, prunedScopes);
+        if(foundScope != null) break; // we can stop as soon as we find the scope in a branch
+      }
+      if(foundScope != null) { // when we found the scopes in my sub-scopes
+        List<IScope> subScopes = new List<IScope>(currentScope.SubScopes); // store the list of sub-scopes
+        prunedScopes.Add(subScopes);
+        // remove all my sub-scopes
+        foreach(IScope subScope in subScopes) { 
+          currentScope.RemoveSubScope(subScope);
+        }
+        // add only the branch that leads to the scope that I search for
+        currentScope.AddSubScope(foundScope);
+        return currentScope; // return that this scope contains the branch that leads to the searched scopes
+      } else {
+        return null; // otherwise we didn't find the searched scope and we can return null
+      }
+    }
+
+    private IScope FindParentScope(IScope currentScope, CompositeOperation compositeOperation) {
+      AtomicOperation currentOperation = (AtomicOperation)compositeOperation.Operations[0];
+      if(currentScope.SubScopes.Contains(currentOperation.Scope)) return currentScope;
+      foreach(IScope subScope in currentScope.SubScopes) {
+        IScope result = FindParentScope(subScope, compositeOperation);
+        if(result != null) return result;
+      }
+      return null;
+    }
+
     private void MergeScope(IScope original, IScope result) {
       // merge the results

trunk/sources/HeuristicLab.DistributedEngine/JobManager.cs

@@ -77 +77 @@
     }
 
-    public WaitHandle BeginExecuteOperation(IOperatorGraph operatorGraph, IScope globalScope, AtomicOperation operation) {
-      ProcessingEngine engine = new ProcessingEngine(operatorGraph, globalScope, operation); // OperatorGraph not needed?
+    public WaitHandle BeginExecuteOperation(IScope globalScope, AtomicOperation operation) {
+      ProcessingEngine engine = new ProcessingEngine(globalScope, operation);
       byte[] zippedEngine = ZipEngine(engine);
       Guid currentEngineGuid = Guid.Empty;

trunk/sources/HeuristicLab.Grid/ProcessingEngine.cs

@@ -39 +39 @@
     }
 
-    public ProcessingEngine(IOperatorGraph graph, IScope globalScope, AtomicOperation initialOperation)
+    public ProcessingEngine(IScope globalScope, AtomicOperation initialOperation)
       : base() {
       this.initialOperation = initialOperation;
       myGlobalScope = globalScope;
-      myOperatorGraph = graph;
       myExecutionStack.Push(initialOperation);
     }