Free cookie consent management tool by TermsFeed Policy Generator

source: misc/tools/HeuristicLab.HiveDrain/HeuristicLab.HiveDrain/JobTaskOneFileDownloader.cs @ 17194

Last change on this file since 17194 was 17194, checked in by mkommend, 4 years ago

#2829: Updated HiveDrain to .Net version 4.6.1 and added handling of invalid characters in filenames.

File size: 5.1 KB
RevLine 
[15342]1#region License Information
2/* HeuristicLab
3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
4 *
5 * This file is part of HeuristicLab.
6 *
7 * HeuristicLab is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
11 *
12 * HeuristicLab is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
19 */
20#endregion
21
22using System;
23using System.Collections.Generic;
[15347]24using System.Linq;
[15342]25using System.Threading;
26using HeuristicLab.Clients.Hive;
27using HeuristicLab.Clients.Hive.Jobs;
28using HeuristicLab.Common;
29using HeuristicLab.Core;
30using HeuristicLab.Optimization;
[15494]31using HeuristicLab.Problems.DataAnalysis;
[15342]32
33namespace HeuristicLab.HiveDrain {
34  public class JobTaskOneFileDownloader {
[17194]35    public string RootLocation { get; set; }
[15342]36
37    public Job ParentJob { get; set; }
38
39    private ILog log;
40
41
42    private static ConcurrentTaskDownloader<ItemTask> downloader =
43        new ConcurrentTaskDownloader<ItemTask>(HeuristicLabHiveDrainApplication.MaxParallelDownloads, HeuristicLabHiveDrainApplication.MaxParallelDownloads);
44
45    private ManualResetEvent allJobsFinished = new ManualResetEvent(false);
46
47    private Semaphore limitSemaphore = null;
48
49    static JobTaskOneFileDownloader() {
50      downloader.ExceptionOccured += downloader_ExceptionOccured;
51    }
52
53    static void downloader_ExceptionOccured(object sender, HeuristicLab.Common.EventArgs<Exception> e) {
54      HiveDrainMainWindow.Log.LogMessage(DateTime.Now.ToShortTimeString() + " ### Exception occured: " + e.Value);
55    }
56
57    public JobTaskOneFileDownloader(string path, Job parentJob, Semaphore sem, ILog log) {
58      RootLocation = path + ".hl";
59      ParentJob = parentJob;
60      limitSemaphore = sem;
61      this.log = log;
62    }
63
64    public void Start() {
65
[15347]66      var allTasks = HiveServiceLocator.Instance.CallHiveService(s => s.GetLightweightJobTasksWithoutStateLog(ParentJob.Id));
[15494]67      var totalJobCount = allTasks.Count;
[15347]68      var optimizers = new List<IOptimizer>();
69      var finishedCount = -1;
70      using (var downloader = new TaskDownloader(allTasks.Select(x => x.Id))) {
71        downloader.StartAsync();
[15342]72
[15350]73        while (!downloader.IsFinished || finishedCount < totalJobCount) {
[15347]74          if (finishedCount != downloader.FinishedCount) {
75            finishedCount = downloader.FinishedCount;
76            log.LogMessage(string.Format("Downloading/deserializing tasks... ({0}/{1} finished)", finishedCount, totalJobCount));
77          }
[15350]78
79          Thread.Sleep(500);
80
81          if (downloader.IsFaulted) {
82            throw downloader.Exception;
83          }
[15347]84        }
[15342]85
[15347]86        IDictionary<Guid, HiveTask> allHiveTasks = downloader.Results;
87        log.LogMessage("Building hive job tree...");
88        var parentTasks = allHiveTasks.Values.Where(x => !x.Task.ParentTaskId.HasValue);
[15342]89
[15347]90        foreach (var parentTask in parentTasks) {
91          BuildHiveJobTree(parentTask, allTasks, allHiveTasks);
[15342]92
[15347]93          var optimizerTask = parentTask.ItemTask as OptimizerTask;
[15342]94
[15347]95          if (optimizerTask != null) {
96            optimizers.Add(optimizerTask.Item);
[15342]97          }
98        }
[15347]99      }
100      if (!optimizers.Any()) return;
101      IStorableContent storable;
102      if (optimizers.Count > 1) {
103        var experiment = new Experiment();
104        experiment.Optimizers.AddRange(optimizers);
105        storable = experiment;
106      } else {
107        var optimizer = optimizers.First();
108        storable = optimizer as IStorableContent;
109      }
110      if (storable != null) {
[15494]111        // remove duplicate datasets
112        log.LogMessage("Removing duplicate datasets...");
113        DatasetUtil.RemoveDuplicateDatasets(storable);
114
[15347]115        log.LogMessage(string.Format("Save job as {0}", RootLocation));
116        ContentManager.Save(storable, RootLocation, true);
117      } else {
118        log.LogMessage(string.Format("Could not save job, content is not storable."));
119      }
120    }
[15342]121
[15347]122    private static void BuildHiveJobTree(HiveTask parentHiveTask, IEnumerable<LightweightTask> allTasks, IDictionary<Guid, HiveTask> allHiveTasks) {
123      IEnumerable<LightweightTask> childTasks = from job in allTasks
124                                                where job.ParentTaskId.HasValue && job.ParentTaskId.Value == parentHiveTask.Task.Id
125                                                orderby job.DateCreated ascending
126                                                select job;
127      foreach (LightweightTask task in childTasks) {
128        HiveTask childHiveTask = allHiveTasks[task.Id];
129        BuildHiveJobTree(childHiveTask, allTasks, allHiveTasks);
130        parentHiveTask.AddChildHiveTask(childHiveTask);
131      }
[15342]132    }
133  }
134}
Note: See TracBrowser for help on using the repository browser.