source: branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobDownloader.cs @ 5535

using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using HeuristicLab.Services.Hive.Common.DataTransfer;

namespace HeuristicLab.Clients.Hive {
  public class HiveJobDownloader {
    private IEnumerable<Guid> jobIds;
    private List<Task<HiveJob>> tasks;
    private bool abort = false;

    public bool IsFinished {
      get {
        return tasks.TrueForAll(t => t.Status == TaskStatus.RanToCompletion ||
                                     t.Status == TaskStatus.Faulted ||
                                     t.Status == TaskStatus.Canceled);
      }
    }

    public int FinishedCount {
      get {
        var faulted = tasks.Where(t => t.Status == TaskStatus.Faulted);
        if (faulted.Count() > 0) {
          abort = true;
          throw faulted.First().Exception;
        }
        return tasks.Count(t => t.Status == TaskStatus.RanToCompletion ||
                                t.Status == TaskStatus.Faulted ||
                                t.Status == TaskStatus.Canceled);
      }
    }

    public IDictionary<Guid, HiveJob> Results {
      get {
        var results = new Dictionary<Guid, HiveJob>();
        foreach (var t in tasks) {
          if (t.Status == TaskStatus.Faulted) {
            throw t.Exception;
          }
          if(t.Result != null) 
            results.Add(t.Result.Job.Id, t.Result);
        }
        return results;
      }
    }

    public HiveJobDownloader(IEnumerable<Guid> jobIds) {
      this.jobIds = jobIds;
    }

    public void StartAsync() {
      abort = false;
      tasks = new List<Task<HiveJob>>();
      TaskScheduler.UnobservedTaskException += new EventHandler<UnobservedTaskExceptionEventArgs>(TaskScheduler_UnobservedTaskException);
      foreach (Guid jobId in jobIds) {
        tasks.Add(Task<JobData>.Factory.StartNew(
          (x) => DownloadJob(x), jobId)
          .ContinueWith((x) => DeserializeJob(x.Result)));
      }
    }

    private void TaskScheduler_UnobservedTaskException(object sender, UnobservedTaskExceptionEventArgs e) {
      e.SetObserved(); // avoid crash of process because task crashes. first exception found is handled in Results property
    }

    // use semaphore to ensure only few concurrenct connections and few SerializedJob objects in memory
    private Semaphore downloadSemaphore = new Semaphore(2, 2);
    private Semaphore deserializeSemaphore = new Semaphore(2, 2);
    protected JobData DownloadJob(object jobId) {
      downloadSemaphore.WaitOne();
      deserializeSemaphore.WaitOne();
      JobData result;
      try {
        if (abort) return null;
        result = ServiceLocator.Instance.CallHiveService(s => s.GetJobData((Guid)jobId));
      }
      finally {
        downloadSemaphore.Release();
      }
      return result;
    }

    protected HiveJob DeserializeJob(JobData jobData) {
      try {
        Job job = ServiceLocator.Instance.CallHiveService(s => s.GetJob(jobData.JobId));
        if (abort || job == null || jobData == null) return null;
        HiveJob hiveJob = new HiveJob(job, jobData, false);
        jobData.Data = null; // reduce memory consumption.
        hiveJob.OptimizerJob.Prepare(); // reduce memory consumption.
        hiveJob.Job = job;
        return hiveJob;
      }
      finally {
        deserializeSemaphore.Release();
      }
    }
  }
}