Changeset 413

Timestamp:

07/31/08 10:07:09 (16 years ago)

Author:

gkronber

Message:

fixed #218 (DistributedEngine silently ignores when grid-jobs are canceled (because of exceptions) on grid-clients) by adding a property that holds an error message to the ProcessingEngine

Location:

trunk/sources

Files:

• HeuristicLab.DistributedEngine/DistributedEngine.cs (modified) (1 diff)
• HeuristicLab.Grid/ProcessingEngine.cs (modified) (5 diffs)

trunk/sources/HeuristicLab.DistributedEngine/DistributedEngine.cs

@@ -127 +127 @@
               }
             }
+
+            CompositeOperation canceledOperations = new CompositeOperation();
+            canceledOperations.ExecuteInParallel = true;
             // retrieve results and merge into scope-tree
             foreach(AtomicOperation parOperation in compositeOperation.Operations) {
               ProcessingEngine resultEngine = jobManager.EndExecuteOperation(parOperation);
               if(resultEngine.Canceled) {
-                // When the engine was canceled because of a problem at the client we can try to execute the steps locally.
-                // If they also fail the (local) distributued-engine will be aborted and we will see an error-message.
-                // so just push the original parallel operation back on the stack to force local execution.
-                ExecutionStack.Push(parOperation);
+                ThreadPool.QueueUserWorkItem(delegate(object state) { OnExceptionOccurred(new JobExecutionException(resultEngine.ErrorMessage)); });
+                canceledOperations.AddOperation(parOperation);
               } else {
                 // if everything went fine we can merge the results into our local scope-tree
                 MergeScope(parOperation.Scope, resultEngine.InitialOperation.Scope);
               }
+            }
+
+            if(canceledOperations.Operations.Count > 0) {
+              myExecutionStack.Push(canceledOperations);
+              Abort();
             }
           } catch(Exception e) {

trunk/sources/HeuristicLab.Grid/ProcessingEngine.cs

@@ -36 +36 @@
     }
 
+    private string errorMessage;
+    public string ErrorMessage {
+      get { return errorMessage; }
+    }
+
     public ProcessingEngine()
       : base() {
@@ -51 +56 @@
       canceledAttr.Value = Canceled.ToString();
       node.Attributes.Append(canceledAttr);
+      if(errorMessage != null) {
+        XmlAttribute errorMessageAttr = document.CreateAttribute("ErrorMessage");
+        errorMessageAttr.Value = ErrorMessage;
+        node.Attributes.Append(errorMessageAttr);
+      }
       node.AppendChild(PersistenceManager.Persist("InitialOperation", initialOperation, document, persistedObjects));
       return node;
@@ -58 +68 @@
       base.Populate(node, restoredObjects);
       myCanceled = bool.Parse(node.Attributes["Canceled"].Value);
+      if(node.Attributes["ErrorMessage"] != null) errorMessage = node.Attributes["ErrorMessage"].Value;
       initialOperation = (AtomicOperation)PersistenceManager.Restore(node.SelectSingleNode("InitialOperation"), restoredObjects);
     }
@@ -69 +80 @@
           next = atomicOperation.Operator.Execute(atomicOperation.Scope);
         } catch(Exception ex) {
-          Trace.TraceWarning("Exception while executing an engine.\n" + ex.Message + "\n" + ex.StackTrace);
+          errorMessage = CreateErrorMessage(ex);
+          Trace.TraceWarning(errorMessage);
           // push operation on stack again
           myExecutionStack.Push(atomicOperation);
@@ -83 +95 @@
       }
     }
+
+    private string CreateErrorMessage(Exception ex) {
+      StringBuilder sb = new StringBuilder();
+      sb.Append("Sorry, but something went wrong!\n\n" + ex.Message + "\n\n" + ex.StackTrace);
+
+      while(ex.InnerException != null) {
+        ex = ex.InnerException;
+        sb.Append("\n\n-----\n\n" + ex.Message + "\n\n" + ex.StackTrace);
+      }
+      return sb.ToString();
+    }
   }
 }