Changeset 15347


Ignore:
Timestamp:
09/06/17 10:07:50 (2 years ago)
Author:
bburlacu
Message:

#2829: Adapt the JobTaskOneFileDownloader to save an experiment instead of a run collection.

Location:
trunk/tools/HeuristicLab.HiveDrain/HeuristicLab.HiveDrain
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • trunk/tools/HeuristicLab.HiveDrain/HeuristicLab.HiveDrain/JobDownloader.cs

    r15342 r15347  
    5252
    5353      foreach (Job j in jobsLoaded) {
    54         if ((!String.IsNullOrEmpty(NamePattern) && j.Name.Contains(NamePattern)) ||
    55             String.IsNullOrEmpty(NamePattern)) {
     54        if (string.IsNullOrEmpty(NamePattern) || j.Name.Contains(NamePattern)) {
    5655          string jobPath = Path.Combine(RootLocation, String.Format("{0} - {1}", j.Name, j.Id));
    5756          log.LogMessage(String.Format("\"{0}\": {1}", j.Name, j.Id));
  • trunk/tools/HeuristicLab.HiveDrain/HeuristicLab.HiveDrain/JobTaskOneFileDownloader.cs

    r15342 r15347  
    2222using System;
    2323using System.Collections.Generic;
     24using System.Linq;
    2425using System.Threading;
    2526using HeuristicLab.Clients.Hive;
     
    6869      results = new RunCollection();
    6970
    70       IEnumerable<LightweightTask> allTasks;
    71       allTasks = HiveServiceLocator.Instance.CallHiveService(s =>
    72           s.GetLightweightJobTasksWithoutStateLog(ParentJob.Id));
     71      var allTasks = HiveServiceLocator.Instance.CallHiveService(s => s.GetLightweightJobTasksWithoutStateLog(ParentJob.Id));
     72      var totalJobCount = allTasks.Count();
     73      var optimizers = new List<IOptimizer>();
     74      var finishedCount = -1;
     75      using (var downloader = new TaskDownloader(allTasks.Select(x => x.Id))) {
     76        downloader.StartAsync();
    7377
    74       foreach (var lightTask in allTasks) {
    75         if (lightTask.State == TaskState.Finished) {
    76           AddDownloaderTask(lightTask.Id);
    77           log.LogMessage(String.Format("   Getting Id {0}: {1}", lightTask.Id, DateTime.Now.ToShortTimeString()));
    78         } else
    79           log.LogMessage(String.Format("   {0} => ignored ({1})", lightTask.Id, lightTask.State.ToString()));
     78        while (!downloader.IsFinished) {
     79          if (finishedCount != downloader.FinishedCount) {
     80            finishedCount = downloader.FinishedCount;
     81            log.LogMessage(string.Format("Downloading/deserializing tasks... ({0}/{1} finished)", finishedCount, totalJobCount));
     82          }
     83        }
     84
     85        IDictionary<Guid, HiveTask> allHiveTasks = downloader.Results;
     86        log.LogMessage("Building hive job tree...");
     87        var parentTasks = allHiveTasks.Values.Where(x => !x.Task.ParentTaskId.HasValue);
     88
     89        foreach (var parentTask in parentTasks) {
     90          BuildHiveJobTree(parentTask, allTasks, allHiveTasks);
     91
     92          var optimizerTask = parentTask.ItemTask as OptimizerTask;
     93
     94          if (optimizerTask != null) {
     95            optimizers.Add(optimizerTask.Item);
     96          }
     97        }
    8098      }
    81       endReached = true;
    82       if (jobCount == 0)
    83         allJobsFinished.Set();
    84 
    85       allJobsFinished.WaitOne();
    86       log.LogMessage("Saving data to file...");
    87       ContentManager.Save(results, RootLocation, true);
    88 
    89       GC.Collect();
    90       log.LogMessage(String.Format("All tasks for job {0} finished", ParentJob.Name));
     99      if (!optimizers.Any()) return;
     100      IStorableContent storable;
     101      if (optimizers.Count > 1) {
     102        var experiment = new Experiment();
     103        experiment.Optimizers.AddRange(optimizers);
     104        storable = experiment;
     105      } else {
     106        var optimizer = optimizers.First();
     107        storable = optimizer as IStorableContent;
     108      }
     109      if (storable != null) {
     110        log.LogMessage(string.Format("Save job as {0}", RootLocation));
     111        ContentManager.Save(storable, RootLocation, true);
     112      } else {
     113        log.LogMessage(string.Format("Could not save job, content is not storable."));
     114      }
    91115    }
    92116
    93     /// <summary>
    94     /// adds a task with state finished to the downloader
    95     /// </summary>
    96     /// <param name="taskId"></param>
    97     /// <param name="taskPath"></param>
    98     private void AddDownloaderTask(Guid taskId) {
    99       //wait for free slot
    100       limitSemaphore.WaitOne();
    101 
    102       Interlocked.Increment(ref jobCount);
    103       downloader.DownloadTaskDataAndTask(taskId, (task, itemTask) => {
    104 
    105         log.LogMessage(String.Format("\"{0}\" - [{1}]: {2} finished", ParentJob.Name, task.Id, itemTask.Name));
    106         if (itemTask is OptimizerTask) {
    107           OptimizerTask optimizerTask = itemTask as OptimizerTask;
    108           IOptimizer opt = (IOptimizer)optimizerTask.Item;
    109 
    110           lock (results) {
    111             results.AddRange(opt.Runs);
    112           }
    113 
    114           log.LogMessage(String.Format("\"{0}\" - [{1}]: {2} added to result collection", ParentJob.Name, task.Id, itemTask.Name));
    115         } else {
    116           log.LogMessage(String.Format("Unsupported task type {0}", itemTask.GetType().Name));
    117         }
    118 
    119         limitSemaphore.Release();
    120         Interlocked.Decrement(ref jobCount);
    121 
    122         //if this was the last job
    123         if (jobCount == 0 && endReached)
    124           allJobsFinished.Set();
    125       });
     117    private static void BuildHiveJobTree(HiveTask parentHiveTask, IEnumerable<LightweightTask> allTasks, IDictionary<Guid, HiveTask> allHiveTasks) {
     118      IEnumerable<LightweightTask> childTasks = from job in allTasks
     119                                                where job.ParentTaskId.HasValue && job.ParentTaskId.Value == parentHiveTask.Task.Id
     120                                                orderby job.DateCreated ascending
     121                                                select job;
     122      foreach (LightweightTask task in childTasks) {
     123        HiveTask childHiveTask = allHiveTasks[task.Id];
     124        BuildHiveJobTree(childHiveTask, allTasks, allHiveTasks);
     125        parentHiveTask.AddChildHiveTask(childHiveTask);
     126      }
    126127    }
    127128  }
Note: See TracChangeset for help on using the changeset viewer.