Changeset 6178

Timestamp:

05/10/11 17:58:59 (14 years ago)

Author:

cneumuel

Message:

#1233

• added semaphores to ensure an appdomain is never unloaded when the start method has not finished
• HiveEngine uploading and downloading of jobs works and is displayed in the view

Location:

branches/HeuristicLab.Hive-3.4

Files:

• binaries/HeuristicLab.Hive.ExperimentManager-3.4.zip (modified) (previous)
• sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs (modified) (4 diffs)
• sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs (modified) (1 diff)
• sources/HeuristicLab.Clients.Hive.Slave/3.4/PluginCache.cs (modified) (1 diff)
• sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/HiveJobView.cs (modified) (9 diffs)
• sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/OptimizerHiveJobView.cs (modified) (1 diff)
• sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/RefreshableHiveExperimentView.cs (modified) (5 diffs)
• sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/TreeView/DeleteJobTreeNodeAction.cs (modified) (1 diff)
• sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/TreeView/HiveJobItemTreeView.cs (modified) (3 diffs)
• sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/TreeView/NewExperimentTreeNodeAction.cs (modified) (1 diff)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/ConcurrentJobDownloader.cs (added)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/ConcurrentJobDownloaderException.cs (added)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/EngineHiveJob.cs (added)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/ExperimentManagerClient.cs (modified) (1 diff)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJob.cs (modified) (16 diffs)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobDownloader.cs (modified) (6 diffs)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/JobDownloader.cs (deleted)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/JobDownloaderException.cs (deleted)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/OptimizerHiveJob.cs (modified) (15 diffs)
• sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/RefreshableHiveExperiment.cs (modified) (8 diffs)
• sources/HeuristicLab.Clients.Hive/3.4/HeuristicLab.Clients.Hive-3.4.csproj (modified) (2 diffs)
• sources/HeuristicLab.Clients.Hive/3.4/Jobs/EngineJob.cs (modified) (2 diffs)
• sources/HeuristicLab.Clients.Hive/3.4/ScopeExtensions.cs (added)
• sources/HeuristicLab.HiveEngine.Test/Program.cs (modified) (1 diff)
• sources/HeuristicLab.HiveEngine/3.4/HiveEngine.cs (modified) (6 diffs)
• sources/HeuristicLab.Services.Hive.Web/Hive-3.4/Web.config (modified) (2 diffs)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs

@@ -47 +47 @@
     private Dictionary<Guid, Executor> executors = new Dictionary<Guid, Executor>();
     private Dictionary<Guid, AppDomain> appDomains = new Dictionary<Guid, AppDomain>();
+
+    // signalizes if the Executor.Start method has properly finished. only then the appdomain may be unloaded
+    private Dictionary<Guid, Semaphore> semaphores = new Dictionary<Guid, Semaphore>();
 
     private WcfService wcfService;
@@ -484 +487 @@
             executor.MemoryNeeded = job.MemoryNeeded;
             clientCom.LogMessage("Starting Executor for job " + job.Id);
-            executor.Start(jobData.Data);
             lock (executors) {
               executors.Add(job.Id, executor);
             }
+            semaphores[job.Id] = new Semaphore(0, 1);
+            executor.Start(jobData.Data);
+            semaphores[job.Id].Release();
           }
           catch (Exception exception) {
@@ -535 +540 @@
     /// Kill a appdomain with a specific id.
     /// </summary>
-    /// <param name="id">the GUID of the job</param>    
-    public void KillAppDomain(Guid id) {
+    /// <param name="jobId">the GUID of the job</param>    
+    public void KillAppDomain(Guid jobId) {
       if (Thread.CurrentThread.ManagedThreadId != this.coreThreadId) {
-        EnqueueExecutorMessage<Guid>(KillAppDomain, id);
+        EnqueueExecutorMessage<Guid>(KillAppDomain, jobId);
         return;
       }
 
-      clientCom.LogMessage("Shutting down Appdomain for Job " + id);
+      clientCom.LogMessage("Shutting down Appdomain for Job " + jobId);
       lock (executors) {
         try {
-          if (executors.ContainsKey(id)) {
-            executors[id].Dispose();
-            executors.Remove(id);
-          }
-
-          if (appDomains.ContainsKey(id)) {
-            appDomains[id].UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
-
+          if (executors.ContainsKey(jobId)) {
+            executors[jobId].Dispose();
+            executors.Remove(jobId);
+          }
+
+          if (appDomains.ContainsKey(jobId)) {
+            appDomains[jobId].UnhandledException -= new UnhandledExceptionEventHandler(AppDomain_UnhandledException);
             int repeat = 5;
             while (repeat > 0) {
               try {
-                AppDomain.Unload(appDomains[id]);
+                semaphores[jobId].WaitOne();
+                AppDomain.Unload(appDomains[jobId]);
+                semaphores[jobId].Dispose();
+                semaphores.Remove(jobId);
                 repeat = 0;
               }
@@ -569 +576 @@
               }
             }
-            appDomains.Remove(id);
-          }
-
-          PluginCache.Instance.DeletePluginsForJob(id);
+            appDomains.Remove(jobId);
+          }
+
+          PluginCache.Instance.DeletePluginsForJob(jobId);
           GC.Collect();
         }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Executor.cs

@@ -89 +89 @@
         } else {
           Job.Start();
-          startJobSem.WaitOne();
+          if (!startJobSem.WaitOne(TimeSpan.FromSeconds(15))) {
+            throw new TimeoutException("Timeout when starting the job. JobStarted event was not fired.");
+          }
         }
       }
       catch (Exception e) {
         this.currentException = e;
-        Job_JobFailed(this, new HeuristicLab.Common.EventArgs<Exception>(e));
+        Job_JobFailed(this, new EventArgs<Exception>(e));
       }
     }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/PluginCache.cs

@@ -278 +278 @@
     internal void DeletePluginsForJob(Guid id) {
       try {
-        SlaveClientCom.Instance.ClientCom.LogMessage("unloading...");
+        SlaveClientCom.Instance.ClientCom.LogMessage("Deleting plugins...");
         int tries = 5;
         while (tries > 0) {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/HiveJobView.cs

@@ -61 +61 @@
       base.OnContentChanged();
       if (Content != null && Content.Job != null) {
-        computeInParallelCheckBox.Checked = Content.JobItem.ComputeInParallel;
+        computeInParallelCheckBox.Checked = Content.ItemJob.ComputeInParallel;
       } else {
         computeInParallelCheckBox.Checked = false;
@@ -72 +72 @@
     protected virtual void RegisterJobEvents() {
       if (Content != null && Content.Job != null) {
-        Content.JobItem.ComputeInParallelChanged += new EventHandler(OptimizerJob_ComputeInParallelChanged);
-        Content.JobItem.ItemChanged += new EventHandler(Job_ItemChanged);
+        Content.ItemJob.ComputeInParallelChanged += new EventHandler(OptimizerJob_ComputeInParallelChanged);
+        Content.ItemJob.ItemChanged += new EventHandler(Job_ItemChanged);
       }
     }
@@ -79 +79 @@
     protected virtual void DeregisterJobEvents() {
       if (Content != null && Content.Job != null) {
-        Content.JobItem.ComputeInParallelChanged -= new EventHandler(OptimizerJob_ComputeInParallelChanged);
-        Content.JobItem.ItemChanged -= new EventHandler(Job_ItemChanged);
+        Content.ItemJob.ComputeInParallelChanged -= new EventHandler(OptimizerJob_ComputeInParallelChanged);
+        Content.ItemJob.ItemChanged -= new EventHandler(Job_ItemChanged);
       }
     }
@@ -111 +111 @@
 
     protected virtual void Job_ItemChanged(object sender, EventArgs e) {
-      if (Content != null && Content.Job != null && Content.JobItem.Item != null) {
-        optimizerItemView.Content = Content.JobItem.Item;
+      if (Content != null && Content.Job != null && Content.ItemJob.Item != null) {
+        optimizerItemView.Content = Content.ItemJob.Item;
       } else {
         optimizerItemView.Content = null;
@@ -129 +129 @@
           this.exceptionTextBox.Text = Content.Job.CurrentStateLog != null ? Content.Job.CurrentStateLog.Exception : string.Empty;
           this.lastUpdatedTextBox.Text = Content.Job.LastJobDataUpdate.ToString();
-          if (Content.JobItem.ComputeInParallel) {
+          if (Content.ItemJob.ComputeInParallel) {
             this.stateLogViewHost.Content = new StateLogListList(
                 this.Content.ChildHiveJobs.Select(child => new StateLogList(child.Job.StateLog)
@@ -165 +165 @@
       this.coresNeededTextBox.ReadOnly = this.ReadOnly;
       this.memoryNeededTextBox.ReadOnly = this.ReadOnly;
-      this.computeInParallelCheckBox.Enabled = !this.ReadOnly && this.Content != null && this.Content.JobItem != null && this.Content.JobItem.IsParallelizable;
-
-      this.modifyItemButton.Enabled = (Content != null && Content.JobItem.Item != null && (Content.Job.State == JobState.Paused || Content.Job.State == JobState.Offline || Content.Job.State == JobState.Finished || Content.Job.State == JobState.Failed || Content.Job.State == JobState.Aborted));
+      this.computeInParallelCheckBox.Enabled = !this.ReadOnly && this.Content != null && this.Content.ItemJob != null && this.Content.ItemJob.IsParallelizable;
+
+      this.modifyItemButton.Enabled = (Content != null && Content.ItemJob.Item != null && (Content.Job.State == JobState.Paused || Content.Job.State == JobState.Offline || Content.Job.State == JobState.Finished || Content.Job.State == JobState.Failed || Content.Job.State == JobState.Aborted));
 
       optimizerItemView.ReadOnly = true;
@@ -176 +176 @@
         Invoke(new EventHandler(OptimizerJob_ComputeInParallelChanged), sender, e);
       } else {
-        computeInParallelCheckBox.Checked = Content.JobItem.ComputeInParallel;
+        computeInParallelCheckBox.Checked = Content.ItemJob.ComputeInParallel;
       }
     }
@@ -183 +183 @@
     #region Child Control Events
      protected virtual void computeInParallelCheckBox_CheckedChanged(object sender, EventArgs e) {
-      if (Content != null && Content.JobItem != null) {
-        this.Content.JobItem.ComputeInParallel = this.computeInParallelCheckBox.Checked;
+      if (Content != null && Content.ItemJob != null) {
+        this.Content.ItemJob.ComputeInParallel = this.computeInParallelCheckBox.Checked;
       }
     }
@@ -196 +196 @@
 
     protected virtual void modifyItemButton_Click(object sender, EventArgs e) {
-      MainFormManager.MainForm.ShowContent(Content.JobItem.Item);
+      MainFormManager.MainForm.ShowContent(Content.ItemJob.Item);
     }
     #endregion

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/OptimizerHiveJobView.cs

@@ -43 +43 @@
 
     protected override void Job_ItemChanged(object sender, EventArgs e) {
-      if (Content != null && Content.Job != null && Content.JobItem.Item != null) {
-        runCollectionViewHost.Content = Content.JobItem.Item.Runs;
+      if (Content != null && Content.Job != null && Content.ItemJob.Item != null) {
+        runCollectionViewHost.Content = Content.ItemJob.Item.Runs;
       } else {
         runCollectionViewHost.Content = null;

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/RefreshableHiveExperimentView.cs

@@ -55 +55 @@
     }
 
-    protected override void DeregisterContentEvents() {
-      Content.RefreshAutomaticallyChanged -= new EventHandler(Content_RefreshAutomaticallyChanged);
-      Content.PropertyChanged -= new System.ComponentModel.PropertyChangedEventHandler(Content_PropertyChanged);
-      Content.HiveExperimentChanged -= new EventHandler(Content_HiveExperimentChanged);      
-      base.DeregisterContentEvents();
-    }
-
     protected override void RegisterContentEvents() {
       base.RegisterContentEvents();
@@ -67 +60 @@
       Content.PropertyChanged += new System.ComponentModel.PropertyChangedEventHandler(Content_PropertyChanged);
       Content.HiveExperimentChanged += new EventHandler(Content_HiveExperimentChanged);
+      Content.IsControllableChanged += new EventHandler(Content_IsControllableChanged);
+    }
+    protected override void DeregisterContentEvents() {
+      Content.RefreshAutomaticallyChanged -= new EventHandler(Content_RefreshAutomaticallyChanged);
+      Content.PropertyChanged -= new System.ComponentModel.PropertyChangedEventHandler(Content_PropertyChanged);
+      Content.HiveExperimentChanged -= new EventHandler(Content_HiveExperimentChanged);
+      Content.IsControllableChanged -= new EventHandler(Content_IsControllableChanged);
+      base.DeregisterContentEvents();
     }
 
@@ -134 +135 @@
         bool jobsLoaded = Content.HiveExperiment.HiveJobs != null && Content.HiveExperiment.HiveJobs.All(x => x.Job.Id != Guid.Empty);
 
-        this.nameTextBox.ReadOnly = Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded;
-        this.resourceNamesTextBox.ReadOnly = Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded;
-        this.jobsTreeView.ReadOnly = Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded;
-        this.useLocalPluginsCheckBox.Enabled = !(Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded);
-        this.refreshAutomaticallyCheckBox.Enabled = alreadyUploaded && jobsLoaded && Content.HiveExperiment.ExecutionState == ExecutionState.Started;
-        this.refreshButton.Enabled = alreadyUploaded;
-
-        this.Locked = Content.HiveExperiment.ExecutionState == ExecutionState.Started;
+        this.nameTextBox.ReadOnly =                 !Content.IsControllable || Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded;
+        this.resourceNamesTextBox.ReadOnly =        !Content.IsControllable || Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded;
+        this.jobsTreeView.ReadOnly =                !Content.IsControllable || Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded;
+
+        this.useLocalPluginsCheckBox.Enabled =      Content.IsControllable && !(Content.HiveExperiment.ExecutionState != ExecutionState.Prepared || alreadyUploaded);
+        this.refreshAutomaticallyCheckBox.Enabled = Content.IsControllable && alreadyUploaded && jobsLoaded && Content.HiveExperiment.ExecutionState == ExecutionState.Started;
+        this.refreshButton.Enabled =                Content.IsControllable && alreadyUploaded;
+        this.Locked =                               !Content.IsControllable || Content.HiveExperiment.ExecutionState == ExecutionState.Started;
       }
       SetEnabledStateOfExecutableButtons();
@@ -246 +247 @@
       }
     }
+    private void Content_IsControllableChanged(object sender, EventArgs e) {
+      SetEnabledStateOfControls();
+    }
     #endregion
 
@@ -302 +306 @@
         startButton.Enabled = pauseButton.Enabled = stopButton.Enabled = resetButton.Enabled = false;
       } else {
-        startButton.Enabled = Content.HiveExperiment.HiveJobs != null && Content.HiveExperiment.HiveJobs.Count > 0 && Content.HiveExperiment.ExecutionState == ExecutionState.Prepared;
-        pauseButton.Enabled = Content.HiveExperiment.ExecutionState == ExecutionState.Started;
-        stopButton.Enabled = Content.HiveExperiment.ExecutionState == ExecutionState.Started;
+        startButton.Enabled = Content.IsControllable && Content.HiveExperiment.HiveJobs != null && Content.HiveExperiment.HiveJobs.Count > 0 && Content.HiveExperiment.ExecutionState == ExecutionState.Prepared;
+        pauseButton.Enabled = Content.IsControllable && Content.HiveExperiment.ExecutionState == ExecutionState.Started;
+        stopButton.Enabled = Content.IsControllable && Content.HiveExperiment.ExecutionState == ExecutionState.Started;
         resetButton.Enabled = false;
       }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/TreeView/DeleteJobTreeNodeAction.cs

@@ -24 +24 @@
         hiveJobs.Remove(item);
       } else {
-        var experiment = parentItem.JobItem.Item as Experiment;
+        var experiment = parentItem.ItemJob.Item as Experiment;
         if (experiment != null) {
-          experiment.Optimizers.Remove(((OptimizerJob)item.JobItem).Item);
+          experiment.Optimizers.Remove(((OptimizerJob)item.ItemJob).Item);
         }
       }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/TreeView/HiveJobItemTreeView.cs

@@ -81 +81 @@
           Content.Add(new OptimizerHiveJob(optimizer));
         } else {
-          var experiment = ((HiveJob)treeView.SelectedNode.Tag).JobItem.Item as Experiment;
+          var experiment = ((HiveJob)treeView.SelectedNode.Tag).ItemJob.Item as Experiment;
           if (experiment != null) {
             experiment.Optimizers.Add(optimizer);
@@ -96 +96 @@
           Content.Remove((HiveJob)treeView.SelectedNode.Tag);
         } else {
-          var experiment = parentItem.JobItem.Item as Experiment;
+          var experiment = parentItem.ItemJob.Item as Experiment;
           if (experiment != null) {
-            experiment.Optimizers.Remove(((OptimizerJob)selectedItem.JobItem).Item);
+            experiment.Optimizers.Remove(((OptimizerJob)selectedItem.ItemJob).Item);
           }
         }        
@@ -108 +108 @@
       var actions = base.GetTreeNodeItemActions(selectedItem);
       if (selectedItem != null) {
-        if (selectedItem.JobItem.Item is Experiment) {
+        if (selectedItem.ItemJob.Item is Experiment) {
 
         }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Views/3.4/ExperimentManager/TreeView/NewExperimentTreeNodeAction.cs

@@ -43 +43 @@
     public void Execute(HiveJob item, HiveJob parentItem) {
       if (item != null) {
-        var experiment = item.JobItem.Item as Experiment;
+        var experiment = item.ItemJob.Item as Experiment;
         experiment.Optimizers.Add(new Experiment());
       } else {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/ExperimentManagerClient.cs

@@ -274 +274 @@
       List<IPluginDescription> plugins;
 
-      if (hiveJob.JobItem.ComputeInParallel &&
-        (hiveJob.JobItem.Item is Optimization.Experiment || hiveJob.JobItem.Item is Optimization.BatchRun)) {
+      if (hiveJob.ItemJob.ComputeInParallel &&
+        (hiveJob.ItemJob.Item is Optimization.Experiment || hiveJob.ItemJob.Item is Optimization.BatchRun)) {
         hiveJob.Job.IsParentJob = true;
         hiveJob.Job.FinishWhenChildJobsFinished = true;
-        hiveJob.JobItem.CollectChildJobs = false; // don't collect child-jobs on slaves
+        hiveJob.ItemJob.CollectChildJobs = false; // don't collect child-jobs on slaves
         jobData = hiveJob.GetAsJobData(true, out plugins);
       } else {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJob.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Drawing;
 using System.Linq;
@@ -55 +56 @@
     }
 
+    [Storable]
     protected Job job;
     public Job Job {
@@ -60 +62 @@
       set {
         if (job != value) {
+          DeregisterJobEvents();
           job = value;
+          RegisterJobEvents();
+          itemJobDownloaded = false;
           OnJobChanged();
           OnToStringChanged();
           OnItemImageChanged();
         }
+        
       }
     }
 
     [Storable]
-    protected ItemJob jobItem;
-    public ItemJob JobItem {
-      get { return jobItem; }
+    protected ItemJob itemJob;
+    public ItemJob ItemJob {
+      get { return itemJob; }
       internal set {
-        if (jobItem != null && syncJobsWithOptimizers) {
+        if (itemJob != null && syncJobsWithOptimizers) {
           this.childHiveJobs.Clear();
         }
-        if (jobItem != value) {
-          DergisterJobItemsEvents();
-          jobItem = value;
-          RegisterJobItemEvents();
+        if (itemJob != value) {
+          DergisterItemJobEvents();
+          itemJob = value;
+          RegisterItemJobEvents();
           OnJobItemChanged();
-        }
-      }
+          itemJobDownloaded = true;
+        }
+      }
+    }
+
+    // job downloaded since last status change
+    [Storable]
+    private bool itemJobDownloaded = false;
+    public bool ItemJobDownloaded {
+      get { return itemJobDownloaded; } 
     }
 
@@ -106 +120 @@
     }
 
-    public HiveJob(ItemJob jobItem, bool autoCreateChildHiveJobs)
+    public HiveJob(ItemJob itemJob, bool autoCreateChildHiveJobs)
       : this() {
       this.syncJobsWithOptimizers = autoCreateChildHiveJobs;
-      this.JobItem = jobItem;
+      this.ItemJob = itemJob;
       this.syncJobsWithOptimizers = true;
     }
@@ -117 +131 @@
       this.Job = job;
       try {
-        this.JobItem = PersistenceUtil.Deserialize<ItemJob>(jobData.Data);
+        this.ItemJob = PersistenceUtil.Deserialize<ItemJob>(jobData.Data);
       }
       catch {
-        this.JobItem = null;
+        this.ItemJob = null;
       }
       this.childHiveJobs = new ItemList<HiveJob>();
@@ -130 +144 @@
       : base(original, cloner) {
       this.Job = cloner.Clone(original.job);
-      this.JobItem = cloner.Clone(original.JobItem);
+      this.ItemJob = cloner.Clone(original.ItemJob);
       this.childHiveJobs = cloner.Clone(original.childHiveJobs);
       this.syncJobsWithOptimizers = original.syncJobsWithOptimizers;
+      this.itemJobDownloaded = original.itemJobDownloaded;
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -141 +156 @@
     protected virtual void UpdateChildHiveJobs() { }
 
-    protected virtual void RegisterJobItemEvents() {
-      if (JobItem != null) {
-        JobItem.ComputeInParallelChanged += new EventHandler(JobItem_ComputeInParallelChanged);
-        JobItem.ToStringChanged += new EventHandler(JobItem_ToStringChanged);
-      }
-    }
-    protected virtual void DergisterJobItemsEvents() {
-      if (JobItem != null) {
-        JobItem.ComputeInParallelChanged -= new EventHandler(JobItem_ComputeInParallelChanged);
-        JobItem.ToStringChanged -= new EventHandler(JobItem_ToStringChanged);
+    protected virtual void RegisterItemJobEvents() {
+      if (ItemJob != null) {
+        ItemJob.ComputeInParallelChanged += new EventHandler(JobItem_ComputeInParallelChanged);
+        ItemJob.ToStringChanged += new EventHandler(JobItem_ToStringChanged);
+      }
+    }
+    protected virtual void DergisterItemJobEvents() {
+      if (ItemJob != null) {
+        ItemJob.ComputeInParallelChanged -= new EventHandler(JobItem_ComputeInParallelChanged);
+        ItemJob.ToStringChanged -= new EventHandler(JobItem_ToStringChanged);
       }
     }
@@ -168 +183 @@
       this.OnToStringChanged();
     }
-    
+
     protected virtual void JobItem_ComputeInParallelChanged(object sender, EventArgs e) {
-      if (JobItem != null && syncJobsWithOptimizers) {
+      if (ItemJob != null && syncJobsWithOptimizers) {
         this.UpdateChildHiveJobs();
       }
@@ -180 +195 @@
 
     public override string ToString() {
-      if (jobItem != null) {
-        return jobItem.ToString();
+      if (itemJob != null) {
+        return itemJob.ToString();
       } else {
         return base.ToString();
@@ -196 +211 @@
         job.Command = lightweightJob.Command;
         job.LastJobDataUpdate = lightweightJob.LastJobDataUpdate;
-        
+
         OnJobStateChanged();
         OnToStringChanged();
@@ -210 +225 @@
     /// </param>
     public virtual JobData GetAsJobData(bool withoutChildOptimizers, out List<IPluginDescription> plugins) {
-      plugins = null;
-      return null;
+      plugins = new List<IPluginDescription>();
+      if (this.itemJob == null)
+        return null;
+
+      IEnumerable<Type> usedTypes;
+      byte[] jobByteArray = PersistenceUtil.Serialize(this.ItemJob, out usedTypes);
+
+      JobData jobData = new JobData() {
+        JobId = job.Id,
+        Data = jobByteArray
+      };
+
+      PluginUtil.CollectDeclaringPlugins(plugins, usedTypes);
+
+      return jobData;
     }
 
@@ -239 +267 @@
       if (handler != null) handler(this, EventArgs.Empty);
     }
+
+    private void RegisterJobEvents() {
+      if (job != null)
+        job.PropertyChanged += new PropertyChangedEventHandler(job_PropertyChanged);
+    }
+
+    private void DeregisterJobEvents() {
+      if (job != null)
+        job.PropertyChanged += new System.ComponentModel.PropertyChangedEventHandler(job_PropertyChanged);
+    }
+
+    private void job_PropertyChanged(object sender, PropertyChangedEventArgs e) {
+      if (e.PropertyName == "State") {
+        itemJobDownloaded = false;
+      }
+    }
     #endregion
-    
+
     /// <summary>
     /// Returns a list of HiveJobs including this and all its child-jobs recursively
@@ -326 +370 @@
       } else {
         ServiceLocator.Instance.CallHiveService(s => s.PauseJob(this.job.Id));
-      }      
+      }
     }
 
@@ -343 +387 @@
         JobData jobData = new JobData();
         jobData.JobId = this.job.Id;
-        jobData.Data = PersistenceUtil.Serialize(this.jobItem);
+        jobData.Data = PersistenceUtil.Serialize(this.itemJob);
         service.UpdateJobData(this.Job, jobData);
         service.RestartJob(this.job.Id);
@@ -350 +394 @@
       });
     }
-    
+
     public ICollection<IItemTreeNodeAction<HiveJob>> Actions {
       get {
@@ -364 +408 @@
   public class HiveJob<T> : HiveJob where T : ItemJob {
 
-    public new T JobItem {
-      get { return (T)base.JobItem; }
-      internal set { base.JobItem = value; }
+    public new T ItemJob {
+      get { return (T)base.ItemJob; }
+      internal set { base.ItemJob = value; }
     }
 
     #region Constructors and Cloning
     public HiveJob() : base() { }
+    public HiveJob(T itemJob) : base(itemJob, true) { }
     protected HiveJob(HiveJob original, Cloner cloner)
       : base(original, cloner) {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobDownloader.cs

@@ -30 +30 @@
   public class HiveJobDownloader {
     private IEnumerable<Guid> jobIds;
-    private JobDownloader<ItemJob> jobDownloader;
+    private ConcurrentJobDownloader<ItemJob> jobDownloader;
     private IDictionary<Guid, HiveJob> results;
 
     public bool IsFinished {
       get {
-        //return tasks.TrueForAll(t => t.Status == TaskStatus.RanToCompletion ||
-        //                             t.Status == TaskStatus.Faulted ||
-        //                             t.Status == TaskStatus.Canceled);
         return results.Count == jobIds.Count();
       }
@@ -44 +41 @@
     public int FinishedCount {
       get {
-        //var faulted = tasks.Where(t => t.Status == TaskStatus.Faulted);
-        //if (faulted.Count() > 0) {
-        //  abort = true;
-        //  throw faulted.First().Exception;
-        //}
-        //return tasks.Count(t => t.Status == TaskStatus.RanToCompletion ||
-        //                        t.Status == TaskStatus.Faulted ||
-        //                        t.Status == TaskStatus.Canceled);
         return results.Count;
       }
@@ -58 +47 @@
     public IDictionary<Guid, HiveJob> Results {
       get {
-        //var results = new Dictionary<Guid, HiveJob>();
-        //foreach (var t in tasks) {
-        //  if (t.Status == TaskStatus.Faulted) {
-        //    throw t.Exception;
-        //  }
-        //  if (t.Result != null)
-        //    results.Add(t.Result.Job.Id, t.Result);
-        //}
         return results;
       }
@@ -72 +53 @@
     public HiveJobDownloader(IEnumerable<Guid> jobIds) {
       this.jobIds = jobIds;
-      this.jobDownloader = new JobDownloader<ItemJob>(2, 2);
+      this.jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
       this.results = new Dictionary<Guid, HiveJob>();
     }
@@ -81 +62 @@
           (id, itemJob, exception) => {
             if (exception != null) {
-              throw new JobDownloaderException("Downloading job failed", exception);
+              throw new ConcurrentJobDownloaderException("Downloading job failed", exception);
             }
             Job job = ServiceLocator.Instance.CallHiveService(s => s.GetJob(id));
@@ -95 +76 @@
             }
           });
-      }
-
-      //tasks = new List<Task<HiveJob>>();
-      //TaskScheduler.UnobservedTaskException += new EventHandler<UnobservedTaskExceptionEventArgs>(TaskScheduler_UnobservedTaskException);
-      //foreach (Guid jobId in jobIds) {
-      //  tasks.Add(Task<JobData>.Factory.StartNew(
-      //    (x) => DownloadJob(x), jobId)
-      //    .ContinueWith((x) => DeserializeJob(x.Result)));
-      //}
+      }      
     }
-
-    //private void TaskScheduler_UnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs e) {
-    //  e.SetObserved(); // avoid crash of process because task crashes. first exception found is handled in Results property
-    //}
-
-    //// use semaphore to ensure only few concurrenct connections and few SerializedJob objects in memory
-    //private Semaphore downloadSemaphore = new Semaphore(2, 2);
-    //private Semaphore deserializeSemaphore = new Semaphore(2, 2);
-    //protected JobData DownloadJob(object jobId) {
-    //  downloadSemaphore.WaitOne();
-    //  deserializeSemaphore.WaitOne();
-    //  JobData result;
-    //  try {
-    //    if (abort) return null;
-    //    result = ServiceLocator.Instance.CallHiveService(s => s.GetJobData((Guid)jobId));
-    //  }
-    //  finally {
-    //    downloadSemaphore.Release();
-    //  }
-    //  return result;
-    //}
-
-    //protected HiveJob DeserializeJob(JobData jobData) {
-    //  try {
-    //    Job job = ServiceLocator.Instance.CallHiveService(s => s.GetJob(jobData.JobId));
-    //    if (abort || job == null || jobData == null) return null;
-
-    //    HiveJob hiveJob;
-    //    var itemJob = PersistenceUtil.Deserialize<ItemJob>(jobData.Data);
-    //    if (itemJob is OptimizerJob) {
-    //      hiveJob = new OptimizerHiveJob((OptimizerJob)itemJob);
-    //    } else {
-    //      hiveJob = new HiveJob(itemJob, true);
-    //    }
-    //    jobData.Data = null; // reduce memory consumption.
-    //    hiveJob.Job = job;
-    //    return hiveJob;
-    //  }
-    //  finally {
-    //    deserializeSemaphore.Release();
-    //  }
-    //}
   }
 }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/OptimizerHiveJob.cs

@@ -37 +37 @@
     public OptimizerHiveJob(IOptimizer optimizer)
       : this() {
-      this.JobItem = new OptimizerJob(optimizer);
+      this.ItemJob = new OptimizerJob(optimizer);
     }
     public OptimizerHiveJob(OptimizerJob optimizerJob)
       : this() {
-      this.JobItem = optimizerJob;
+      this.ItemJob = optimizerJob;
     }
     protected OptimizerHiveJob(OptimizerHiveJob original, Cloner cloner)
@@ -60 +60 @@
       base.UpdateChildHiveJobs();
       if (Job != null && syncJobsWithOptimizers) {
-        if (!JobItem.ComputeInParallel) {
+        if (!ItemJob.ComputeInParallel) {
           this.childHiveJobs.Clear();
         } else {
-          if (JobItem.Item is Optimization.Experiment) {
-            Optimization.Experiment experiment = (Optimization.Experiment)JobItem.Item;
+          if (ItemJob.Item is Optimization.Experiment) {
+            Optimization.Experiment experiment = (Optimization.Experiment)ItemJob.Item;
             foreach (IOptimizer childOpt in experiment.Optimizers) {
               this.childHiveJobs.Add(new OptimizerHiveJob(childOpt));
             }
-          } else if (JobItem.Item is Optimization.BatchRun) {
-            Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
+          } else if (ItemJob.Item is Optimization.BatchRun) {
+            Optimization.BatchRun batchRun = ItemJob.OptimizerAsBatchRun;
             if (batchRun.Optimizer != null) {
               while (this.childHiveJobs.Count < batchRun.Repetitions) {
@@ -83 +83 @@
     }
 
-    protected override void RegisterJobItemEvents() {
-      base.RegisterJobItemEvents();
-      if (JobItem != null) {
-        if (JobItem.Item is Optimization.Experiment) {
-          Optimization.Experiment experiment = JobItem.OptimizerAsExperiment;
+    protected override void RegisterItemJobEvents() {
+      base.RegisterItemJobEvents();
+      if (ItemJob != null) {
+        if (ItemJob.Item is Optimization.Experiment) {
+          Optimization.Experiment experiment = ItemJob.OptimizerAsExperiment;
           experiment.Optimizers.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
           experiment.Optimizers.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
           experiment.Optimizers.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
           experiment.Optimizers.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
-        } else if (JobItem.Item is Optimization.BatchRun) {
-          Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
+        } else if (ItemJob.Item is Optimization.BatchRun) {
+          Optimization.BatchRun batchRun = ItemJob.OptimizerAsBatchRun;
           batchRun.RepetitionsChanged += new EventHandler(batchRun_RepetitionsChanged);
           batchRun.OptimizerChanged += new EventHandler(batchRun_OptimizerChanged);
@@ -99 +99 @@
       }
     }
-    protected override void DergisterJobItemsEvents() {
-      base.DergisterJobItemsEvents();
-      if (JobItem != null) {
-        if (JobItem.Item is Optimization.Experiment) {
-          Optimization.Experiment experiment = JobItem.OptimizerAsExperiment;
+    protected override void DergisterItemJobEvents() {
+      base.DergisterItemJobEvents();
+      if (ItemJob != null) {
+        if (ItemJob.Item is Optimization.Experiment) {
+          Optimization.Experiment experiment = ItemJob.OptimizerAsExperiment;
           experiment.Optimizers.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
           experiment.Optimizers.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
           experiment.Optimizers.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
           experiment.Optimizers.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
-        } else if (JobItem.Item is Optimization.BatchRun) {
-          Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
+        } else if (ItemJob.Item is Optimization.BatchRun) {
+          Optimization.BatchRun batchRun = ItemJob.OptimizerAsBatchRun;
           batchRun.RepetitionsChanged -= new EventHandler(batchRun_RepetitionsChanged);
           batchRun.OptimizerChanged -= new EventHandler(batchRun_OptimizerChanged);
@@ -129 +129 @@
 
     private void Optimizers_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.Items) {
           if (GetChildByOptimizer(item.Value) == null && item.Value.Name != "Placeholder") {
@@ -138 +138 @@
     }
     private void Optimizers_ItemsReplaced(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.OldItems) {
           this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
@@ -150 +150 @@
     }
     private void Optimizers_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.Items) {
           this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
@@ -157 +157 @@
     }
     private void Optimizers_CollectionReset(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.Items) {
           this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
@@ -174 +174 @@
       syncJobsWithOptimizers = false; // don't sync with optimizers during this method
 
-      if (this.JobItem != null && this.JobItem.Item != null) {
-        if (this.JobItem.Item is Optimization.Experiment) {
-          UpdateOptimizerInExperiment(this.JobItem.OptimizerAsExperiment, optimizerJob);
-        } else if (this.JobItem.Item is Optimization.BatchRun) {
-          UpdateOptimizerInBatchRun(this.JobItem.OptimizerAsBatchRun, optimizerJob);
+      if (this.ItemJob != null && this.ItemJob.Item != null) {
+        if (this.ItemJob.Item is Optimization.Experiment) {
+          UpdateOptimizerInExperiment(this.ItemJob.OptimizerAsExperiment, optimizerJob);
+        } else if (this.ItemJob.Item is Optimization.BatchRun) {
+          UpdateOptimizerInBatchRun(this.ItemJob.OptimizerAsBatchRun, optimizerJob);
         }
       }
@@ -185 +185 @@
       if (!optimizerJob.ComputeInParallel) {
         child.syncJobsWithOptimizers = false;
-        child.JobItem = optimizerJob;
+        child.ItemJob = optimizerJob;
         child.syncJobsWithOptimizers = true;
       }
@@ -236 +236 @@
     internal void SetIndexInParentOptimizerList(OptimizerHiveJob parentHiveJob) {
       if (parentHiveJob != null) {
-        if (parentHiveJob.JobItem.Item is Optimization.Experiment) {
-          this.JobItem.IndexInParentOptimizerList = parentHiveJob.JobItem.OptimizerAsExperiment.Optimizers.IndexOf(this.JobItem.Item);
-        } else if (parentHiveJob.JobItem.Item is Optimization.BatchRun) {
-          this.JobItem.IndexInParentOptimizerList = 0;
+        if (parentHiveJob.ItemJob.Item is Optimization.Experiment) {
+          this.ItemJob.IndexInParentOptimizerList = parentHiveJob.ItemJob.OptimizerAsExperiment.Optimizers.IndexOf(this.ItemJob.Item);
+        } else if (parentHiveJob.ItemJob.Item is Optimization.BatchRun) {
+          this.ItemJob.IndexInParentOptimizerList = 0;
         } else {
           throw new NotSupportedException("Only Experiment and BatchRuns are supported");
@@ -253 +253 @@
       var optimizerHiveJob = (OptimizerHiveJob)hiveJob;
       syncJobsWithOptimizers = false;
-      if (this.JobItem != null && optimizerHiveJob.JobItem != null) {
+      if (this.ItemJob != null && optimizerHiveJob.ItemJob != null) {
         // if job is in state Paused, it has to preserve its ResultCollection, which is cleared when a optimizer is added to an experiment
         OptimizerJob optimizerJobClone = null;
         if (optimizerHiveJob.Job.State == JobState.Paused) {
-          optimizerJobClone = (OptimizerJob)optimizerHiveJob.JobItem.Clone();
-        }
-
-        if (this.JobItem.Item is Optimization.Experiment) {
-          if (!this.JobItem.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.JobItem.Item)) {
-            UpdateOptimizerInExperiment(this.JobItem.OptimizerAsExperiment, optimizerHiveJob.JobItem);
+          optimizerJobClone = (OptimizerJob)optimizerHiveJob.ItemJob.Clone();
+        }
+
+        if (this.ItemJob.Item is Optimization.Experiment) {
+          if (!this.ItemJob.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.ItemJob.Item)) {
+            UpdateOptimizerInExperiment(this.ItemJob.OptimizerAsExperiment, optimizerHiveJob.ItemJob);
           }
-        } else if (this.JobItem.Item is Optimization.BatchRun) {
-          UpdateOptimizerInBatchRun(this.JobItem.OptimizerAsBatchRun, optimizerHiveJob.JobItem);
+        } else if (this.ItemJob.Item is Optimization.BatchRun) {
+          UpdateOptimizerInBatchRun(this.ItemJob.OptimizerAsBatchRun, optimizerHiveJob.ItemJob);
         }
 
         if (optimizerHiveJob.Job.State == JobState.Paused) {
-          optimizerHiveJob.JobItem = optimizerJobClone;
+          optimizerHiveJob.ItemJob = optimizerJobClone;
         }
       }
@@ -283 +283 @@
     public override JobData GetAsJobData(bool withoutChildOptimizers, out List<IPluginDescription> plugins) {
       plugins = new List<IPluginDescription>();
-      if (this.jobItem == null) // || this.jobItem.Optimizer == null
+      if (this.itemJob == null) // || this.jobItem.Optimizer == null
         return null;
 
       IEnumerable<Type> usedTypes;
       byte[] jobByteArray;
-      if (withoutChildOptimizers && this.JobItem.Item is Optimization.Experiment) {
-        OptimizerJob clonedJob = (OptimizerJob)this.JobItem.Clone(); // use a cloned job, so that the childHiveJob don't get confused
+      if (withoutChildOptimizers && this.ItemJob.Item is Optimization.Experiment) {
+        OptimizerJob clonedJob = (OptimizerJob)this.ItemJob.Clone(); // use a cloned job, so that the childHiveJob don't get confused
         clonedJob.OptimizerAsExperiment.Optimizers.Clear();
         jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
-      } else if (withoutChildOptimizers && this.JobItem.Item is Optimization.BatchRun) {
-        OptimizerJob clonedJob = (OptimizerJob)this.JobItem.Clone();
+      } else if (withoutChildOptimizers && this.ItemJob.Item is Optimization.BatchRun) {
+        OptimizerJob clonedJob = (OptimizerJob)this.ItemJob.Clone();
         clonedJob.OptimizerAsBatchRun.Optimizer = null;
         jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
-      } else if (this.JobItem.Item is IAlgorithm) {
-        ((IAlgorithm)this.JobItem.Item).StoreAlgorithmInEachRun = false; // avoid storing the algorithm in runs to reduce size
-        jobByteArray = PersistenceUtil.Serialize(this.JobItem, out usedTypes);
+      } else if (this.ItemJob.Item is IAlgorithm) {
+        ((IAlgorithm)this.ItemJob.Item).StoreAlgorithmInEachRun = false; // avoid storing the algorithm in runs to reduce size
+        jobByteArray = PersistenceUtil.Serialize(this.ItemJob, out usedTypes);
       } else {
-        jobByteArray = PersistenceUtil.Serialize(this.JobItem, out usedTypes);
+        jobByteArray = PersistenceUtil.Serialize(this.ItemJob, out usedTypes);
       }
 
@@ -315 +315 @@
     public OptimizerHiveJob GetChildByOptimizerJob(OptimizerJob optimizerJob) {
       foreach (OptimizerHiveJob child in ChildHiveJobs) {
-        if (child.JobItem == optimizerJob)
+        if (child.ItemJob == optimizerJob)
           return child;
       }
@@ -323 +323 @@
     public HiveJob<OptimizerJob> GetChildByOptimizer(IOptimizer optimizer) {
       foreach (OptimizerHiveJob child in ChildHiveJobs) {
-        if (child.JobItem.Item == optimizer)
+        if (child.ItemJob.Item == optimizer)
           return child;
       }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/RefreshableHiveExperiment.cs

@@ -34 +34 @@
   public class RefreshableHiveExperiment : IHiveItem, IDeepCloneable, IContent, IProgressReporter {
     private JobResultPoller jobResultPoller;
-    private JobDownloader<ItemJob> jobDownloader = new JobDownloader<ItemJob>(2, 2);
+    private ConcurrentJobDownloader<ItemJob> jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
     private static object locker = new object();
 
@@ -83 +83 @@
     }
 
+    [Storable]
+    private bool isControllable = true;
+    public bool IsControllable {
+      get { return isControllable; }
+      set {
+        if (value != isControllable) {
+          isControllable = value;
+          OnIsControllableChanged();
+        }
+      }
+    }
+
     #region Constructors and Cloning
     public RefreshableHiveExperiment() {
@@ -99 +111 @@
       this.RefreshAutomatically = original.RefreshAutomatically;
       this.IncludeJobs = original.IncludeJobs;
+      this.IsControllable = original.IsControllable;
     }
     public IDeepCloneable Clone(Cloner cloner) {
@@ -163 +176 @@
             jobDownloader.DownloadJob(hj.Job.Id, (id, itemJob, exception) => {
               if (exception != null) {
-                throw new JobDownloaderException("Downloading job failed.", exception);
+                throw new ConcurrentJobDownloaderException("Downloading job failed.", exception);
               }
 
@@ -171 +184 @@
                 // if the job is paused, download but don't integrate into parent optimizer (to avoid Prepare)
                 if (hj.Job.State == JobState.Paused) {
-                  hj.JobItem = itemJob;
+                  hj.ItemJob = itemJob;
                 } else {
                   if (lightweightJob.ParentJobId.HasValue) {
@@ -177 +190 @@
                     parentHiveJob.IntegrateChild(itemJob, hj.Job.Id);
                   } else {
-                    hj.JobItem = itemJob;
+                    hj.ItemJob = itemJob;
                   }
                 }
@@ -211 +224 @@
 
     public bool AllJobsFinished() {
-      return hiveExperiment.GetAllHiveJobs().All(j => j.Job.State == JobState.Finished
-                                            || j.Job.State == JobState.Aborted
-                                            || j.Job.State == JobState.Failed);
+      return hiveExperiment.GetAllHiveJobs().All(j => (j.Job.State == JobState.Finished
+                                                   || j.Job.State == JobState.Aborted
+                                                   || j.Job.State == JobState.Failed)
+                                                   && j.ItemJobDownloaded);
     }
 
@@ -299 +313 @@
       if (handler != null) handler(sender, e);
     }
+
+    public event EventHandler IsControllableChanged;
+    private void OnIsControllableChanged() {
+      var handler = IsControllableChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
     #endregion
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/HeuristicLab.Clients.Hive-3.4.csproj

@@ -120 +120 @@
     <Compile Include="Exceptions\JobResultPollingException.cs" />
     <Compile Include="Exceptions\OptimizerNotFoundException.cs" />
-    <Compile Include="ExperimentManager\JobDownloader.cs" />
+    <Compile Include="ExperimentManager\ConcurrentJobDownloader.cs" />
+    <Compile Include="ExperimentManager\EngineHiveJob.cs" />
     <Compile Include="ExperimentManager\ExperimentManagerClient.cs" />
     <Compile Include="ExperimentManager\HiveJobDownloader.cs" />
-    <Compile Include="ExperimentManager\JobDownloaderException.cs" />
+    <Compile Include="ExperimentManager\ConcurrentJobDownloaderException.cs" />
     <Compile Include="ExperimentManager\TreeView\IItemTree.cs" />
     <Compile Include="ExperimentManager\TreeView\IItemTreeAction.cs" />
@@ -145 +146 @@
     <None Include="Properties\AssemblyInfo.cs.frame" />
     <Compile Include="Exceptions\ServiceClientFactoryException.cs" />
+    <Compile Include="ScopeExtensions.cs" />
     <Compile Include="ServiceClients\Appointment.cs" />
     <Compile Include="ServiceClients\HiveItemCollection.cs" />

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/Jobs/EngineJob.cs

@@ -72 +72 @@
 
     protected override void RegisterItemEvents() {
+      base.RegisterItemEvents();
       Item.Stopped += new EventHandler(engine_Stopped);
+      Item.Paused += new EventHandler(Item_Paused);
+      Item.Started += new EventHandler(Item_Started);
       Item.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(engine_ExceptionOccurred);
+      Item.ExecutionStateChanged += new EventHandler(Item_ExecutionStateChanged);
+      Item.ExecutionTimeChanged += new EventHandler(Item_ExecutionTimeChanged);
     }
 
     protected override void DeregisterItemEvents() {
       Item.Stopped -= new EventHandler(engine_Stopped);
+      Item.Paused -= new EventHandler(Item_Paused);
+      Item.Started -= new EventHandler(Item_Started);
       Item.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(engine_ExceptionOccurred);
+      Item.ExecutionStateChanged -= new EventHandler(Item_ExecutionStateChanged);
+      Item.ExecutionTimeChanged -= new EventHandler(Item_ExecutionTimeChanged);
+      base.DeregisterItemEvents();
     }
 
@@ -87 +97 @@
     private void engine_Stopped(object sender, EventArgs e) {
       OnJobStopped();
+    }
+
+    private void Item_Paused(object sender, EventArgs e) {
+      OnJobPaused();
+    }
+
+    private void Item_ExecutionTimeChanged(object sender, EventArgs e) {
+      OnExecutionTimeChanged();
+    }
+
+    private void Item_ExecutionStateChanged(object sender, EventArgs e) {
+      OnExecutionStateChanged();
+    }
+
+    private void Item_Started(object sender, EventArgs e) {
+      OnJobStarted();
     }
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine.Test/Program.cs

@@ -48 +48 @@
       GeneticAlgorithm ga = new GeneticAlgorithm();
       ga.Problem = new SingleObjectiveTestFunctionProblem();
-      ga.Engine = new HiveEngine();      
-      ga.PopulationSize.Value = 3;
+      ga.Engine = new HiveEngine();
+      ga.Elites.Value = 0;
+      ga.PopulationSize.Value = 4;
       ga.MaximumGenerations.Value = 3;
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.HiveEngine/3.4/HiveEngine.cs

@@ -7 +7 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Hive;
 using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
 using HeuristicLab.PluginInfrastructure;
@@ -140 +141 @@
 
             IScope[] scopes = ExecuteOnHive(jobs, parentScopeClone, cancellationToken);
-            
+            //IScope[] scopes = ExecuteLocally(jobs, parentScopeClone, cancellationToken);
+
             for (int i = 0; i < coll.Count; i++) {
               if (coll[i] is IAtomicOperation) {
@@ -203 +205 @@
 
     // testfunction:
-    //private IScope[] ExecuteLocally(EngineJob[] jobs, IScope parentScopeClone, CancellationToken cancellationToken) {
-    //  IScope[] scopes = new Scope[jobs.Length];
-    //  for (int i = 0; i < jobs.Length; i++) {
-    //    var job = (EngineJob)jobs[i].Clone();
-    //    job.Start();
-    //    while (job.ExecutionState != ExecutionState.Stopped) {
-    //      Thread.Sleep(100);
-    //    }
-    //    scopes[i] = ((IAtomicOperation)job.InitialOperation).Scope;
-    //  }
-    //  return scopes;
-    //}
+    private IScope[] ExecuteLocally(EngineJob[] jobs, IScope parentScopeClone, CancellationToken cancellationToken) {
+      IScope[] scopes = new Scope[jobs.Length];
+      for (int i = 0; i < jobs.Length; i++) {
+        var serialized = PersistenceUtil.Serialize(jobs[i]);
+        var deserialized = PersistenceUtil.Deserialize<IJob>(serialized);
+        deserialized.Start();
+        while (deserialized.ExecutionState != ExecutionState.Stopped) {
+          Thread.Sleep(100);
+        }
+        var serialized2 = PersistenceUtil.Serialize(deserialized);
+        var deserialized2 = PersistenceUtil.Deserialize<EngineJob>(serialized2);
+        var newScope = ((IAtomicOperation)deserialized2.InitialOperation).Scope;
+        scopes[i] = newScope;
+      }
+      return scopes;
+    }
 
     /// <summary>
@@ -230 +236 @@
       try {
         List<Guid> remainingJobIds = new List<Guid>();
-        List<LightweightJob> lightweightJobs;
-
-        int finishedCount = 0;
-        int uploadCount = 0;
 
         // create hive experiment
         hiveExperiment.Name = "HiveEngine Run " + hiveExperiments.Count;
+        hiveExperiment.DateCreated = DateTime.Now;
         hiveExperiment.UseLocalPlugins = this.UseLocalPlugins;
         hiveExperiment.ResourceNames = this.ResourceNames;
-        hiveExperiment.Id = ServiceLocator.Instance.CallHiveService(s => s.AddHiveExperiment(hiveExperiment));
         var refreshableHiveExperiment = new RefreshableHiveExperiment(hiveExperiment);
+        refreshableHiveExperiment.IsControllable = false;
         hiveExperiments.Add(refreshableHiveExperiment);
 
@@ -246 +249 @@
         var uploadTasks = new List<Task<Job>>();
         for (int i = 0; i < jobs.Length; i++) {
-          var job = jobs[i];
+          hiveExperiment.HiveJobs.Add(new EngineHiveJob(jobs[i], parentScopeClone));
 
           // shuffle random variable to avoid the same random sequence in each operation; todo: does not yet work (it cannot find the random variable)
-          IRandom random = FindRandomParameter(job.InitialOperation as IExecutionContext);
+          IRandom random = FindRandomParameter(jobs[i].InitialOperation as IExecutionContext);
           if (random != null)
             random.Reset(random.Next());
-
-          uploadTasks.Add(Task.Factory.StartNew<Job>((keyValuePairObj) => {
-            return UploadJob(keyValuePairObj, parentScopeClone, cancellationToken, GetResourceIds(), hiveExperiment.Id);
-          }, new KeyValuePair<int, EngineJob>(i, job), cancellationToken));
-        }
-
-        Task processUploadedJobsTask = new Task(() => {
-          // process finished upload-tasks
-          int uploadTasksCount = uploadTasks.Count;
-          for (int i = 0; i < uploadTasksCount; i++) {
-            cancellationToken.ThrowIfCancellationRequested();
-
-            var uploadTasksArray = uploadTasks.ToArray();
-            var task = uploadTasksArray[Task.WaitAny(uploadTasksArray)];
-            if (task.Status == TaskStatus.Faulted) {
-              LogException(task.Exception);
-              throw task.Exception;
-            }
-
-            int key = ((KeyValuePair<int, EngineJob>)task.AsyncState).Key;
-            Job job = task.Result;
-            lock (locker) {
-              uploadCount++;
-              jobIndices.Add(job.Id, key);
-              remainingJobIds.Add(job.Id);
-            }
-            jobs[key] = null; // relax memory
-            LogMessage(string.Format("Uploaded job #{0}", key + 1, job.Id));
-            uploadTasks.Remove(task);
-          }
-        }, cancellationToken, TaskCreationOptions.PreferFairness);
-        processUploadedJobsTask.Start();
-
-        refreshableHiveExperiment.RefreshAutomatically = true;
-        
+        }
+        ExperimentManagerClient.StartExperiment((e) => { throw e; }, refreshableHiveExperiment);
+        // do polling until experiment is finished and all jobs are downloaded
         while (!refreshableHiveExperiment.AllJobsFinished()) {
-          Thread.Sleep(1000);
-          // update time
-          // handle cancellation
-        }
-
-
-
-        // poll job-statuses and create tasks for those which are finished
-        var downloadTasks = new List<Task<EngineJob>>();
-        var executionTimes = new List<TimeSpan>();
-        var executionTimeOnHiveBefore = executionTimeOnHive;
-        while (processUploadedJobsTask.Status != TaskStatus.RanToCompletion || remainingJobIds.Count > 0) {
-          cancellationToken.ThrowIfCancellationRequested();
-
-          Thread.Sleep(10000);
-          try {
-            lightweightJobs = ServiceLocator.Instance.CallHiveService(s => s.GetLightweightJobs(remainingJobIds));
-
-            var jobsFinished = lightweightJobs.Where(j => j.State == JobState.Finished || j.State == JobState.Failed || j.State == JobState.Aborted);
-            finishedCount += jobsFinished.Count();
-            if (jobsFinished.Count() > 0) LogMessage(string.Format("Finished: {0}/{1}", finishedCount, jobs.Length));
-            ExecutionTimeOnHive = executionTimeOnHiveBefore + executionTimes.Sum() + lightweightJobs.Select(x => x.ExecutionTime.HasValue ? x.ExecutionTime.Value : TimeSpan.Zero).Sum();
-
-            foreach (var result in jobsFinished) {
-              if (result.State == JobState.Finished) {
-                downloadTasks.Add(Task.Factory.StartNew<EngineJob>((jobIdObj) => {
-                  return DownloadJob(jobIndices, jobIdObj, cancellationToken);
-                }, result.Id, cancellationToken));
-              } else if (result.State == JobState.Aborted) {
-                LogMessage(string.Format("Job #{0} aborted (id: {1})", jobIndices[result.Id] + 1, result.Id));
-              } else if (result.State == JobState.Failed) {
-                LogMessage(string.Format("Job #{0} failed (id: {1}): {2}", jobIndices[result.Id] + 1, result.Id, result.CurrentStateLog != null ? result.CurrentStateLog.Exception : string.Empty));
-              }
-              remainingJobIds.Remove(result.Id);
-              executionTimes.Add(result.ExecutionTime.HasValue ? result.ExecutionTime.Value : TimeSpan.Zero);
-            }
-          }
-          catch (Exception e) {
-            LogException(e);
-          }
-        }
-
-        // process finished download-tasks
-        int downloadTasksCount = downloadTasks.Count;
-        for (int i = 0; i < downloadTasksCount; i++) {
-          cancellationToken.ThrowIfCancellationRequested();
-
-          var downloadTasksArray = downloadTasks.ToArray();
-          var task = downloadTasksArray[Task.WaitAny(downloadTasksArray)];
-          var jobId = (Guid)task.AsyncState;
-          if (task.Status == TaskStatus.Faulted) {
-            LogException(task.Exception);
-            throw task.Exception;
-          }
-          scopes[jobIndices[(Guid)task.AsyncState]] = ((IAtomicOperation)task.Result.InitialOperation).Scope;
-          downloadTasks.Remove(task);
-        }
-
-        LogMessage(string.Format("All jobs finished (TotalExecutionTime: {0}).", executionTimes.Sum()));
+          Thread.Sleep(500);
+          this.ExecutionTimeOnHive = TimeSpan.FromMilliseconds(hiveExperiments.Sum(x => x.HiveExperiment.ExecutionTime.TotalMilliseconds));
+        }
+        LogMessage(string.Format("{0} finished (TotalExecutionTime: {1}).", refreshableHiveExperiment.ToString(), refreshableHiveExperiment.HiveExperiment.ExecutionTime));
+
+        // get scopes
+        int j = 0;
+        foreach (var hiveJob in hiveExperiment.HiveJobs) {
+          var scope = ((IAtomicOperation) ((EngineJob)hiveJob.ItemJob).InitialOperation).Scope;
+          scopes[j++] = scope;
+        }
+        refreshableHiveExperiment.RefreshAutomatically = false;
         DeleteHiveExperiment(hiveExperiment.Id);
-
         return scopes;
       }
@@ -505 +428 @@
   }
 
-  public static class ScopeExtensions {
-    public static void ClearParentScopes(this IScope scope) {
-      scope.ClearParentScopes(null);
-    }
-
-    public static void ClearParentScopes(this IScope scope, IScope childScope) {
-      if (childScope != null) {
-        scope.SubScopes.Clear();
-        scope.SubScopes.Add(childScope);
-      }
-      if (scope.Parent != null)
-        scope.Parent.ClearParentScopes(scope);
-    }
-  }
-
   public static class EnumerableExtensions {
     public static TimeSpan Sum(this IEnumerable<TimeSpan> times) {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Web/Hive-3.4/Web.config

@@ -21 +21 @@
     </roleManager>
     <pages controlRenderingCompatibilityVersion="3.5" clientIDMode="AutoID"/>
+    <httpRuntime maxRequestLength="1048576" />
   </system.web>
   
@@ -79 +80 @@
     </modules>-->
     <directoryBrowse enabled="true"/>
+    
   </system.webServer>