Changeset 6178 for branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4

Timestamp:

05/10/11 17:58:59 (13 years ago)

Author:

cneumuel

Message:

#1233

• added semaphores to ensure an appdomain is never unloaded when the start method has not finished
• HiveEngine uploading and downloading of jobs works and is displayed in the view

Location:

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4

Files:

• ExperimentManager/ConcurrentJobDownloader.cs (added)
• ExperimentManager/ConcurrentJobDownloaderException.cs (added)
• ExperimentManager/EngineHiveJob.cs (added)
• ExperimentManager/ExperimentManagerClient.cs (modified) (1 diff)
• ExperimentManager/HiveJob.cs (modified) (16 diffs)
• ExperimentManager/HiveJobDownloader.cs (modified) (6 diffs)
• ExperimentManager/JobDownloader.cs (deleted)
• ExperimentManager/JobDownloaderException.cs (deleted)
• ExperimentManager/OptimizerHiveJob.cs (modified) (15 diffs)
• ExperimentManager/RefreshableHiveExperiment.cs (modified) (8 diffs)
• HeuristicLab.Clients.Hive-3.4.csproj (modified) (2 diffs)
• Jobs/EngineJob.cs (modified) (2 diffs)
• ScopeExtensions.cs (added)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/ExperimentManagerClient.cs

@@ -274 +274 @@
       List<IPluginDescription> plugins;
 
-      if (hiveJob.JobItem.ComputeInParallel &&
-        (hiveJob.JobItem.Item is Optimization.Experiment || hiveJob.JobItem.Item is Optimization.BatchRun)) {
+      if (hiveJob.ItemJob.ComputeInParallel &&
+        (hiveJob.ItemJob.Item is Optimization.Experiment || hiveJob.ItemJob.Item is Optimization.BatchRun)) {
         hiveJob.Job.IsParentJob = true;
         hiveJob.Job.FinishWhenChildJobsFinished = true;
-        hiveJob.JobItem.CollectChildJobs = false; // don't collect child-jobs on slaves
+        hiveJob.ItemJob.CollectChildJobs = false; // don't collect child-jobs on slaves
         jobData = hiveJob.GetAsJobData(true, out plugins);
       } else {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJob.cs

@@ -22 +22 @@
 using System;
 using System.Collections.Generic;
+using System.ComponentModel;
 using System.Drawing;
 using System.Linq;
@@ -55 +56 @@
     }
 
+    [Storable]
     protected Job job;
     public Job Job {
@@ -60 +62 @@
       set {
         if (job != value) {
+          DeregisterJobEvents();
           job = value;
+          RegisterJobEvents();
+          itemJobDownloaded = false;
           OnJobChanged();
           OnToStringChanged();
           OnItemImageChanged();
         }
+        
       }
     }
 
     [Storable]
-    protected ItemJob jobItem;
-    public ItemJob JobItem {
-      get { return jobItem; }
+    protected ItemJob itemJob;
+    public ItemJob ItemJob {
+      get { return itemJob; }
       internal set {
-        if (jobItem != null && syncJobsWithOptimizers) {
+        if (itemJob != null && syncJobsWithOptimizers) {
           this.childHiveJobs.Clear();
         }
-        if (jobItem != value) {
-          DergisterJobItemsEvents();
-          jobItem = value;
-          RegisterJobItemEvents();
+        if (itemJob != value) {
+          DergisterItemJobEvents();
+          itemJob = value;
+          RegisterItemJobEvents();
           OnJobItemChanged();
-        }
-      }
+          itemJobDownloaded = true;
+        }
+      }
+    }
+
+    // job downloaded since last status change
+    [Storable]
+    private bool itemJobDownloaded = false;
+    public bool ItemJobDownloaded {
+      get { return itemJobDownloaded; } 
     }
 
@@ -106 +120 @@
     }
 
-    public HiveJob(ItemJob jobItem, bool autoCreateChildHiveJobs)
+    public HiveJob(ItemJob itemJob, bool autoCreateChildHiveJobs)
       : this() {
       this.syncJobsWithOptimizers = autoCreateChildHiveJobs;
-      this.JobItem = jobItem;
+      this.ItemJob = itemJob;
       this.syncJobsWithOptimizers = true;
     }
@@ -117 +131 @@
       this.Job = job;
       try {
-        this.JobItem = PersistenceUtil.Deserialize<ItemJob>(jobData.Data);
+        this.ItemJob = PersistenceUtil.Deserialize<ItemJob>(jobData.Data);
       }
       catch {
-        this.JobItem = null;
+        this.ItemJob = null;
       }
       this.childHiveJobs = new ItemList<HiveJob>();
@@ -130 +144 @@
       : base(original, cloner) {
       this.Job = cloner.Clone(original.job);
-      this.JobItem = cloner.Clone(original.JobItem);
+      this.ItemJob = cloner.Clone(original.ItemJob);
       this.childHiveJobs = cloner.Clone(original.childHiveJobs);
       this.syncJobsWithOptimizers = original.syncJobsWithOptimizers;
+      this.itemJobDownloaded = original.itemJobDownloaded;
     }
     public override IDeepCloneable Clone(Cloner cloner) {
@@ -141 +156 @@
     protected virtual void UpdateChildHiveJobs() { }
 
-    protected virtual void RegisterJobItemEvents() {
-      if (JobItem != null) {
-        JobItem.ComputeInParallelChanged += new EventHandler(JobItem_ComputeInParallelChanged);
-        JobItem.ToStringChanged += new EventHandler(JobItem_ToStringChanged);
-      }
-    }
-    protected virtual void DergisterJobItemsEvents() {
-      if (JobItem != null) {
-        JobItem.ComputeInParallelChanged -= new EventHandler(JobItem_ComputeInParallelChanged);
-        JobItem.ToStringChanged -= new EventHandler(JobItem_ToStringChanged);
+    protected virtual void RegisterItemJobEvents() {
+      if (ItemJob != null) {
+        ItemJob.ComputeInParallelChanged += new EventHandler(JobItem_ComputeInParallelChanged);
+        ItemJob.ToStringChanged += new EventHandler(JobItem_ToStringChanged);
+      }
+    }
+    protected virtual void DergisterItemJobEvents() {
+      if (ItemJob != null) {
+        ItemJob.ComputeInParallelChanged -= new EventHandler(JobItem_ComputeInParallelChanged);
+        ItemJob.ToStringChanged -= new EventHandler(JobItem_ToStringChanged);
       }
     }
@@ -168 +183 @@
       this.OnToStringChanged();
     }
-    
+
     protected virtual void JobItem_ComputeInParallelChanged(object sender, EventArgs e) {
-      if (JobItem != null && syncJobsWithOptimizers) {
+      if (ItemJob != null && syncJobsWithOptimizers) {
         this.UpdateChildHiveJobs();
       }
@@ -180 +195 @@
 
     public override string ToString() {
-      if (jobItem != null) {
-        return jobItem.ToString();
+      if (itemJob != null) {
+        return itemJob.ToString();
       } else {
         return base.ToString();
@@ -196 +211 @@
         job.Command = lightweightJob.Command;
         job.LastJobDataUpdate = lightweightJob.LastJobDataUpdate;
-        
+
         OnJobStateChanged();
         OnToStringChanged();
@@ -210 +225 @@
     /// </param>
     public virtual JobData GetAsJobData(bool withoutChildOptimizers, out List<IPluginDescription> plugins) {
-      plugins = null;
-      return null;
+      plugins = new List<IPluginDescription>();
+      if (this.itemJob == null)
+        return null;
+
+      IEnumerable<Type> usedTypes;
+      byte[] jobByteArray = PersistenceUtil.Serialize(this.ItemJob, out usedTypes);
+
+      JobData jobData = new JobData() {
+        JobId = job.Id,
+        Data = jobByteArray
+      };
+
+      PluginUtil.CollectDeclaringPlugins(plugins, usedTypes);
+
+      return jobData;
     }
 
@@ -239 +267 @@
       if (handler != null) handler(this, EventArgs.Empty);
     }
+
+    private void RegisterJobEvents() {
+      if (job != null)
+        job.PropertyChanged += new PropertyChangedEventHandler(job_PropertyChanged);
+    }
+
+    private void DeregisterJobEvents() {
+      if (job != null)
+        job.PropertyChanged += new System.ComponentModel.PropertyChangedEventHandler(job_PropertyChanged);
+    }
+
+    private void job_PropertyChanged(object sender, PropertyChangedEventArgs e) {
+      if (e.PropertyName == "State") {
+        itemJobDownloaded = false;
+      }
+    }
     #endregion
-    
+
     /// <summary>
     /// Returns a list of HiveJobs including this and all its child-jobs recursively
@@ -326 +370 @@
       } else {
         ServiceLocator.Instance.CallHiveService(s => s.PauseJob(this.job.Id));
-      }      
+      }
     }
 
@@ -343 +387 @@
         JobData jobData = new JobData();
         jobData.JobId = this.job.Id;
-        jobData.Data = PersistenceUtil.Serialize(this.jobItem);
+        jobData.Data = PersistenceUtil.Serialize(this.itemJob);
         service.UpdateJobData(this.Job, jobData);
         service.RestartJob(this.job.Id);
@@ -350 +394 @@
       });
     }
-    
+
     public ICollection<IItemTreeNodeAction<HiveJob>> Actions {
       get {
@@ -364 +408 @@
   public class HiveJob<T> : HiveJob where T : ItemJob {
 
-    public new T JobItem {
-      get { return (T)base.JobItem; }
-      internal set { base.JobItem = value; }
+    public new T ItemJob {
+      get { return (T)base.ItemJob; }
+      internal set { base.ItemJob = value; }
     }
 
     #region Constructors and Cloning
     public HiveJob() : base() { }
+    public HiveJob(T itemJob) : base(itemJob, true) { }
     protected HiveJob(HiveJob original, Cloner cloner)
       : base(original, cloner) {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobDownloader.cs

@@ -30 +30 @@
   public class HiveJobDownloader {
     private IEnumerable<Guid> jobIds;
-    private JobDownloader<ItemJob> jobDownloader;
+    private ConcurrentJobDownloader<ItemJob> jobDownloader;
     private IDictionary<Guid, HiveJob> results;
 
     public bool IsFinished {
       get {
-        //return tasks.TrueForAll(t => t.Status == TaskStatus.RanToCompletion ||
-        //                             t.Status == TaskStatus.Faulted ||
-        //                             t.Status == TaskStatus.Canceled);
         return results.Count == jobIds.Count();
       }
@@ -44 +41 @@
     public int FinishedCount {
       get {
-        //var faulted = tasks.Where(t => t.Status == TaskStatus.Faulted);
-        //if (faulted.Count() > 0) {
-        //  abort = true;
-        //  throw faulted.First().Exception;
-        //}
-        //return tasks.Count(t => t.Status == TaskStatus.RanToCompletion ||
-        //                        t.Status == TaskStatus.Faulted ||
-        //                        t.Status == TaskStatus.Canceled);
         return results.Count;
       }
@@ -58 +47 @@
     public IDictionary<Guid, HiveJob> Results {
       get {
-        //var results = new Dictionary<Guid, HiveJob>();
-        //foreach (var t in tasks) {
-        //  if (t.Status == TaskStatus.Faulted) {
-        //    throw t.Exception;
-        //  }
-        //  if (t.Result != null)
-        //    results.Add(t.Result.Job.Id, t.Result);
-        //}
         return results;
       }
@@ -72 +53 @@
     public HiveJobDownloader(IEnumerable<Guid> jobIds) {
       this.jobIds = jobIds;
-      this.jobDownloader = new JobDownloader<ItemJob>(2, 2);
+      this.jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
       this.results = new Dictionary<Guid, HiveJob>();
     }
@@ -81 +62 @@
           (id, itemJob, exception) => {
             if (exception != null) {
-              throw new JobDownloaderException("Downloading job failed", exception);
+              throw new ConcurrentJobDownloaderException("Downloading job failed", exception);
             }
             Job job = ServiceLocator.Instance.CallHiveService(s => s.GetJob(id));
@@ -95 +76 @@
             }
           });
-      }
-
-      //tasks = new List<Task<HiveJob>>();
-      //TaskScheduler.UnobservedTaskException += new EventHandler<UnobservedTaskExceptionEventArgs>(TaskScheduler_UnobservedTaskException);
-      //foreach (Guid jobId in jobIds) {
-      //  tasks.Add(Task<JobData>.Factory.StartNew(
-      //    (x) => DownloadJob(x), jobId)
-      //    .ContinueWith((x) => DeserializeJob(x.Result)));
-      //}
+      }      
     }
-
-    //private void TaskScheduler_UnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs e) {
-    //  e.SetObserved(); // avoid crash of process because task crashes. first exception found is handled in Results property
-    //}
-
-    //// use semaphore to ensure only few concurrenct connections and few SerializedJob objects in memory
-    //private Semaphore downloadSemaphore = new Semaphore(2, 2);
-    //private Semaphore deserializeSemaphore = new Semaphore(2, 2);
-    //protected JobData DownloadJob(object jobId) {
-    //  downloadSemaphore.WaitOne();
-    //  deserializeSemaphore.WaitOne();
-    //  JobData result;
-    //  try {
-    //    if (abort) return null;
-    //    result = ServiceLocator.Instance.CallHiveService(s => s.GetJobData((Guid)jobId));
-    //  }
-    //  finally {
-    //    downloadSemaphore.Release();
-    //  }
-    //  return result;
-    //}
-
-    //protected HiveJob DeserializeJob(JobData jobData) {
-    //  try {
-    //    Job job = ServiceLocator.Instance.CallHiveService(s => s.GetJob(jobData.JobId));
-    //    if (abort || job == null || jobData == null) return null;
-
-    //    HiveJob hiveJob;
-    //    var itemJob = PersistenceUtil.Deserialize<ItemJob>(jobData.Data);
-    //    if (itemJob is OptimizerJob) {
-    //      hiveJob = new OptimizerHiveJob((OptimizerJob)itemJob);
-    //    } else {
-    //      hiveJob = new HiveJob(itemJob, true);
-    //    }
-    //    jobData.Data = null; // reduce memory consumption.
-    //    hiveJob.Job = job;
-    //    return hiveJob;
-    //  }
-    //  finally {
-    //    deserializeSemaphore.Release();
-    //  }
-    //}
   }
 }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/OptimizerHiveJob.cs

@@ -37 +37 @@
     public OptimizerHiveJob(IOptimizer optimizer)
       : this() {
-      this.JobItem = new OptimizerJob(optimizer);
+      this.ItemJob = new OptimizerJob(optimizer);
     }
     public OptimizerHiveJob(OptimizerJob optimizerJob)
       : this() {
-      this.JobItem = optimizerJob;
+      this.ItemJob = optimizerJob;
     }
     protected OptimizerHiveJob(OptimizerHiveJob original, Cloner cloner)
@@ -60 +60 @@
       base.UpdateChildHiveJobs();
       if (Job != null && syncJobsWithOptimizers) {
-        if (!JobItem.ComputeInParallel) {
+        if (!ItemJob.ComputeInParallel) {
           this.childHiveJobs.Clear();
         } else {
-          if (JobItem.Item is Optimization.Experiment) {
-            Optimization.Experiment experiment = (Optimization.Experiment)JobItem.Item;
+          if (ItemJob.Item is Optimization.Experiment) {
+            Optimization.Experiment experiment = (Optimization.Experiment)ItemJob.Item;
             foreach (IOptimizer childOpt in experiment.Optimizers) {
               this.childHiveJobs.Add(new OptimizerHiveJob(childOpt));
             }
-          } else if (JobItem.Item is Optimization.BatchRun) {
-            Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
+          } else if (ItemJob.Item is Optimization.BatchRun) {
+            Optimization.BatchRun batchRun = ItemJob.OptimizerAsBatchRun;
             if (batchRun.Optimizer != null) {
               while (this.childHiveJobs.Count < batchRun.Repetitions) {
@@ -83 +83 @@
     }
 
-    protected override void RegisterJobItemEvents() {
-      base.RegisterJobItemEvents();
-      if (JobItem != null) {
-        if (JobItem.Item is Optimization.Experiment) {
-          Optimization.Experiment experiment = JobItem.OptimizerAsExperiment;
+    protected override void RegisterItemJobEvents() {
+      base.RegisterItemJobEvents();
+      if (ItemJob != null) {
+        if (ItemJob.Item is Optimization.Experiment) {
+          Optimization.Experiment experiment = ItemJob.OptimizerAsExperiment;
           experiment.Optimizers.ItemsAdded += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
           experiment.Optimizers.ItemsReplaced += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
           experiment.Optimizers.ItemsRemoved += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
           experiment.Optimizers.CollectionReset += new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
-        } else if (JobItem.Item is Optimization.BatchRun) {
-          Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
+        } else if (ItemJob.Item is Optimization.BatchRun) {
+          Optimization.BatchRun batchRun = ItemJob.OptimizerAsBatchRun;
           batchRun.RepetitionsChanged += new EventHandler(batchRun_RepetitionsChanged);
           batchRun.OptimizerChanged += new EventHandler(batchRun_OptimizerChanged);
@@ -99 +99 @@
       }
     }
-    protected override void DergisterJobItemsEvents() {
-      base.DergisterJobItemsEvents();
-      if (JobItem != null) {
-        if (JobItem.Item is Optimization.Experiment) {
-          Optimization.Experiment experiment = JobItem.OptimizerAsExperiment;
+    protected override void DergisterItemJobEvents() {
+      base.DergisterItemJobEvents();
+      if (ItemJob != null) {
+        if (ItemJob.Item is Optimization.Experiment) {
+          Optimization.Experiment experiment = ItemJob.OptimizerAsExperiment;
           experiment.Optimizers.ItemsAdded -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsAdded);
           experiment.Optimizers.ItemsReplaced -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsReplaced);
           experiment.Optimizers.ItemsRemoved -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_ItemsRemoved);
           experiment.Optimizers.CollectionReset -= new CollectionItemsChangedEventHandler<IndexedItem<IOptimizer>>(Optimizers_CollectionReset);
-        } else if (JobItem.Item is Optimization.BatchRun) {
-          Optimization.BatchRun batchRun = JobItem.OptimizerAsBatchRun;
+        } else if (ItemJob.Item is Optimization.BatchRun) {
+          Optimization.BatchRun batchRun = ItemJob.OptimizerAsBatchRun;
           batchRun.RepetitionsChanged -= new EventHandler(batchRun_RepetitionsChanged);
           batchRun.OptimizerChanged -= new EventHandler(batchRun_OptimizerChanged);
@@ -129 +129 @@
 
     private void Optimizers_ItemsAdded(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.Items) {
           if (GetChildByOptimizer(item.Value) == null && item.Value.Name != "Placeholder") {
@@ -138 +138 @@
     }
     private void Optimizers_ItemsReplaced(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.OldItems) {
           this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
@@ -150 +150 @@
     }
     private void Optimizers_ItemsRemoved(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.Items) {
           this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
@@ -157 +157 @@
     }
     private void Optimizers_CollectionReset(object sender, CollectionItemsChangedEventArgs<IndexedItem<IOptimizer>> e) {
-      if (syncJobsWithOptimizers && this.JobItem.ComputeInParallel) {
+      if (syncJobsWithOptimizers && this.ItemJob.ComputeInParallel) {
         foreach (var item in e.Items) {
           this.childHiveJobs.Remove(this.GetChildByOptimizer(item.Value));
@@ -174 +174 @@
       syncJobsWithOptimizers = false; // don't sync with optimizers during this method
 
-      if (this.JobItem != null && this.JobItem.Item != null) {
-        if (this.JobItem.Item is Optimization.Experiment) {
-          UpdateOptimizerInExperiment(this.JobItem.OptimizerAsExperiment, optimizerJob);
-        } else if (this.JobItem.Item is Optimization.BatchRun) {
-          UpdateOptimizerInBatchRun(this.JobItem.OptimizerAsBatchRun, optimizerJob);
+      if (this.ItemJob != null && this.ItemJob.Item != null) {
+        if (this.ItemJob.Item is Optimization.Experiment) {
+          UpdateOptimizerInExperiment(this.ItemJob.OptimizerAsExperiment, optimizerJob);
+        } else if (this.ItemJob.Item is Optimization.BatchRun) {
+          UpdateOptimizerInBatchRun(this.ItemJob.OptimizerAsBatchRun, optimizerJob);
         }
       }
@@ -185 +185 @@
       if (!optimizerJob.ComputeInParallel) {
         child.syncJobsWithOptimizers = false;
-        child.JobItem = optimizerJob;
+        child.ItemJob = optimizerJob;
         child.syncJobsWithOptimizers = true;
       }
@@ -236 +236 @@
     internal void SetIndexInParentOptimizerList(OptimizerHiveJob parentHiveJob) {
       if (parentHiveJob != null) {
-        if (parentHiveJob.JobItem.Item is Optimization.Experiment) {
-          this.JobItem.IndexInParentOptimizerList = parentHiveJob.JobItem.OptimizerAsExperiment.Optimizers.IndexOf(this.JobItem.Item);
-        } else if (parentHiveJob.JobItem.Item is Optimization.BatchRun) {
-          this.JobItem.IndexInParentOptimizerList = 0;
+        if (parentHiveJob.ItemJob.Item is Optimization.Experiment) {
+          this.ItemJob.IndexInParentOptimizerList = parentHiveJob.ItemJob.OptimizerAsExperiment.Optimizers.IndexOf(this.ItemJob.Item);
+        } else if (parentHiveJob.ItemJob.Item is Optimization.BatchRun) {
+          this.ItemJob.IndexInParentOptimizerList = 0;
         } else {
           throw new NotSupportedException("Only Experiment and BatchRuns are supported");
@@ -253 +253 @@
       var optimizerHiveJob = (OptimizerHiveJob)hiveJob;
       syncJobsWithOptimizers = false;
-      if (this.JobItem != null && optimizerHiveJob.JobItem != null) {
+      if (this.ItemJob != null && optimizerHiveJob.ItemJob != null) {
         // if job is in state Paused, it has to preserve its ResultCollection, which is cleared when a optimizer is added to an experiment
         OptimizerJob optimizerJobClone = null;
         if (optimizerHiveJob.Job.State == JobState.Paused) {
-          optimizerJobClone = (OptimizerJob)optimizerHiveJob.JobItem.Clone();
-        }
-
-        if (this.JobItem.Item is Optimization.Experiment) {
-          if (!this.JobItem.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.JobItem.Item)) {
-            UpdateOptimizerInExperiment(this.JobItem.OptimizerAsExperiment, optimizerHiveJob.JobItem);
+          optimizerJobClone = (OptimizerJob)optimizerHiveJob.ItemJob.Clone();
+        }
+
+        if (this.ItemJob.Item is Optimization.Experiment) {
+          if (!this.ItemJob.OptimizerAsExperiment.Optimizers.Contains(optimizerHiveJob.ItemJob.Item)) {
+            UpdateOptimizerInExperiment(this.ItemJob.OptimizerAsExperiment, optimizerHiveJob.ItemJob);
           }
-        } else if (this.JobItem.Item is Optimization.BatchRun) {
-          UpdateOptimizerInBatchRun(this.JobItem.OptimizerAsBatchRun, optimizerHiveJob.JobItem);
+        } else if (this.ItemJob.Item is Optimization.BatchRun) {
+          UpdateOptimizerInBatchRun(this.ItemJob.OptimizerAsBatchRun, optimizerHiveJob.ItemJob);
         }
 
         if (optimizerHiveJob.Job.State == JobState.Paused) {
-          optimizerHiveJob.JobItem = optimizerJobClone;
+          optimizerHiveJob.ItemJob = optimizerJobClone;
         }
       }
@@ -283 +283 @@
     public override JobData GetAsJobData(bool withoutChildOptimizers, out List<IPluginDescription> plugins) {
       plugins = new List<IPluginDescription>();
-      if (this.jobItem == null) // || this.jobItem.Optimizer == null
+      if (this.itemJob == null) // || this.jobItem.Optimizer == null
         return null;
 
       IEnumerable<Type> usedTypes;
       byte[] jobByteArray;
-      if (withoutChildOptimizers && this.JobItem.Item is Optimization.Experiment) {
-        OptimizerJob clonedJob = (OptimizerJob)this.JobItem.Clone(); // use a cloned job, so that the childHiveJob don't get confused
+      if (withoutChildOptimizers && this.ItemJob.Item is Optimization.Experiment) {
+        OptimizerJob clonedJob = (OptimizerJob)this.ItemJob.Clone(); // use a cloned job, so that the childHiveJob don't get confused
         clonedJob.OptimizerAsExperiment.Optimizers.Clear();
         jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
-      } else if (withoutChildOptimizers && this.JobItem.Item is Optimization.BatchRun) {
-        OptimizerJob clonedJob = (OptimizerJob)this.JobItem.Clone();
+      } else if (withoutChildOptimizers && this.ItemJob.Item is Optimization.BatchRun) {
+        OptimizerJob clonedJob = (OptimizerJob)this.ItemJob.Clone();
         clonedJob.OptimizerAsBatchRun.Optimizer = null;
         jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
-      } else if (this.JobItem.Item is IAlgorithm) {
-        ((IAlgorithm)this.JobItem.Item).StoreAlgorithmInEachRun = false; // avoid storing the algorithm in runs to reduce size
-        jobByteArray = PersistenceUtil.Serialize(this.JobItem, out usedTypes);
+      } else if (this.ItemJob.Item is IAlgorithm) {
+        ((IAlgorithm)this.ItemJob.Item).StoreAlgorithmInEachRun = false; // avoid storing the algorithm in runs to reduce size
+        jobByteArray = PersistenceUtil.Serialize(this.ItemJob, out usedTypes);
       } else {
-        jobByteArray = PersistenceUtil.Serialize(this.JobItem, out usedTypes);
+        jobByteArray = PersistenceUtil.Serialize(this.ItemJob, out usedTypes);
       }
 
@@ -315 +315 @@
     public OptimizerHiveJob GetChildByOptimizerJob(OptimizerJob optimizerJob) {
       foreach (OptimizerHiveJob child in ChildHiveJobs) {
-        if (child.JobItem == optimizerJob)
+        if (child.ItemJob == optimizerJob)
           return child;
       }
@@ -323 +323 @@
     public HiveJob<OptimizerJob> GetChildByOptimizer(IOptimizer optimizer) {
       foreach (OptimizerHiveJob child in ChildHiveJobs) {
-        if (child.JobItem.Item == optimizer)
+        if (child.ItemJob.Item == optimizer)
           return child;
       }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/RefreshableHiveExperiment.cs

@@ -34 +34 @@
   public class RefreshableHiveExperiment : IHiveItem, IDeepCloneable, IContent, IProgressReporter {
     private JobResultPoller jobResultPoller;
-    private JobDownloader<ItemJob> jobDownloader = new JobDownloader<ItemJob>(2, 2);
+    private ConcurrentJobDownloader<ItemJob> jobDownloader = new ConcurrentJobDownloader<ItemJob>(2, 2);
     private static object locker = new object();
 
@@ -83 +83 @@
     }
 
+    [Storable]
+    private bool isControllable = true;
+    public bool IsControllable {
+      get { return isControllable; }
+      set {
+        if (value != isControllable) {
+          isControllable = value;
+          OnIsControllableChanged();
+        }
+      }
+    }
+
     #region Constructors and Cloning
     public RefreshableHiveExperiment() {
@@ -99 +111 @@
       this.RefreshAutomatically = original.RefreshAutomatically;
       this.IncludeJobs = original.IncludeJobs;
+      this.IsControllable = original.IsControllable;
     }
     public IDeepCloneable Clone(Cloner cloner) {
@@ -163 +176 @@
             jobDownloader.DownloadJob(hj.Job.Id, (id, itemJob, exception) => {
               if (exception != null) {
-                throw new JobDownloaderException("Downloading job failed.", exception);
+                throw new ConcurrentJobDownloaderException("Downloading job failed.", exception);
               }
 
@@ -171 +184 @@
                 // if the job is paused, download but don't integrate into parent optimizer (to avoid Prepare)
                 if (hj.Job.State == JobState.Paused) {
-                  hj.JobItem = itemJob;
+                  hj.ItemJob = itemJob;
                 } else {
                   if (lightweightJob.ParentJobId.HasValue) {
@@ -177 +190 @@
                     parentHiveJob.IntegrateChild(itemJob, hj.Job.Id);
                   } else {
-                    hj.JobItem = itemJob;
+                    hj.ItemJob = itemJob;
                   }
                 }
@@ -211 +224 @@
 
     public bool AllJobsFinished() {
-      return hiveExperiment.GetAllHiveJobs().All(j => j.Job.State == JobState.Finished
-                                            || j.Job.State == JobState.Aborted
-                                            || j.Job.State == JobState.Failed);
+      return hiveExperiment.GetAllHiveJobs().All(j => (j.Job.State == JobState.Finished
+                                                   || j.Job.State == JobState.Aborted
+                                                   || j.Job.State == JobState.Failed)
+                                                   && j.ItemJobDownloaded);
     }
 
@@ -299 +313 @@
       if (handler != null) handler(sender, e);
     }
+
+    public event EventHandler IsControllableChanged;
+    private void OnIsControllableChanged() {
+      var handler = IsControllableChanged;
+      if (handler != null) handler(this, EventArgs.Empty);
+    }
     #endregion
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/HeuristicLab.Clients.Hive-3.4.csproj

@@ -120 +120 @@
     <Compile Include="Exceptions\JobResultPollingException.cs" />
     <Compile Include="Exceptions\OptimizerNotFoundException.cs" />
-    <Compile Include="ExperimentManager\JobDownloader.cs" />
+    <Compile Include="ExperimentManager\ConcurrentJobDownloader.cs" />
+    <Compile Include="ExperimentManager\EngineHiveJob.cs" />
     <Compile Include="ExperimentManager\ExperimentManagerClient.cs" />
     <Compile Include="ExperimentManager\HiveJobDownloader.cs" />
-    <Compile Include="ExperimentManager\JobDownloaderException.cs" />
+    <Compile Include="ExperimentManager\ConcurrentJobDownloaderException.cs" />
     <Compile Include="ExperimentManager\TreeView\IItemTree.cs" />
     <Compile Include="ExperimentManager\TreeView\IItemTreeAction.cs" />
@@ -145 +146 @@
     <None Include="Properties\AssemblyInfo.cs.frame" />
     <Compile Include="Exceptions\ServiceClientFactoryException.cs" />
+    <Compile Include="ScopeExtensions.cs" />
     <Compile Include="ServiceClients\Appointment.cs" />
     <Compile Include="ServiceClients\HiveItemCollection.cs" />

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/Jobs/EngineJob.cs

@@ -72 +72 @@
 
     protected override void RegisterItemEvents() {
+      base.RegisterItemEvents();
       Item.Stopped += new EventHandler(engine_Stopped);
+      Item.Paused += new EventHandler(Item_Paused);
+      Item.Started += new EventHandler(Item_Started);
       Item.ExceptionOccurred += new EventHandler<EventArgs<Exception>>(engine_ExceptionOccurred);
+      Item.ExecutionStateChanged += new EventHandler(Item_ExecutionStateChanged);
+      Item.ExecutionTimeChanged += new EventHandler(Item_ExecutionTimeChanged);
     }
 
     protected override void DeregisterItemEvents() {
       Item.Stopped -= new EventHandler(engine_Stopped);
+      Item.Paused -= new EventHandler(Item_Paused);
+      Item.Started -= new EventHandler(Item_Started);
       Item.ExceptionOccurred -= new EventHandler<EventArgs<Exception>>(engine_ExceptionOccurred);
+      Item.ExecutionStateChanged -= new EventHandler(Item_ExecutionStateChanged);
+      Item.ExecutionTimeChanged -= new EventHandler(Item_ExecutionTimeChanged);
+      base.DeregisterItemEvents();
     }
 
@@ -87 +97 @@
     private void engine_Stopped(object sender, EventArgs e) {
       OnJobStopped();
+    }
+
+    private void Item_Paused(object sender, EventArgs e) {
+      OnJobPaused();
+    }
+
+    private void Item_ExecutionTimeChanged(object sender, EventArgs e) {
+      OnExecutionTimeChanged();
+    }
+
+    private void Item_ExecutionStateChanged(object sender, EventArgs e) {
+      OnExecutionStateChanged();
+    }
+
+    private void Item_Started(object sender, EventArgs e) {
+      OnJobStarted();
     }