Changeset 17059 for stable/HeuristicLab.Clients.Hive/3.3/HiveClient.cs

Timestamp:

07/04/19 14:28:13 (5 years ago)

Author:

mkommend

Message:

#2839: Merged 16117, 16122, 16173, 16184, 16185, 16186, 16187, 16202, 16203, 16204, 16205, 16208, 16211, 16209, 16211, 16219, 16257, 16247 into stable.

Location:

stable

Files:

• . (modified) (1 prop)
• HeuristicLab.Clients.Hive (modified) (1 prop)
• HeuristicLab.Clients.Hive/3.3/HiveClient.cs (modified) (18 diffs)

stable/HeuristicLab.Clients.Hive/3.3/HiveClient.cs

@@ -1 +1 @@
 #region License Information
 /* HeuristicLab
- * Copyright (C) 2002-2018 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
+ * Copyright (C) 2002-2016 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
  *
  * This file is part of HeuristicLab.
@@ -57 +57 @@
     }
 
+    private IItemList<Project> projects;
+    public IItemList<Project> Projects {
+      get { return projects; }
+    }
+
+    private IItemList<Resource> resources;
+    public IItemList<Resource> Resources {
+      get { return resources; }
+    }
+
+    private Dictionary<Guid, HashSet<Guid>> projectAncestors;
+    public Dictionary<Guid, HashSet<Guid>> ProjectAncestors {
+      get { return projectAncestors; }
+    }
+
+    private Dictionary<Guid, HashSet<Guid>> projectDescendants;
+    public Dictionary<Guid, HashSet<Guid>> ProjectDescendants {
+      get { return projectDescendants; }
+    }
+
+    private Dictionary<Guid, HashSet<Guid>> resourceAncestors;
+    public Dictionary<Guid, HashSet<Guid>> ResourceAncestors {
+      get { return resourceAncestors; }
+    }
+
+    private Dictionary<Guid, HashSet<Guid>> resourceDescendants;
+    public Dictionary<Guid, HashSet<Guid>> ResourceDescendants {
+      get { return resourceDescendants; }
+    }
+
+    private Dictionary<Guid, string> projectNames;
+    public Dictionary<Guid, string> ProjectNames {
+      get { return projectNames; }
+    }
+
+    private HashSet<Project> disabledParentProjects;
+    public HashSet<Project> DisabledParentProjects {
+      get { return disabledParentProjects; }
+    }
+
+    private Dictionary<Guid, string> resourceNames;
+    public Dictionary<Guid, string> ResourceNames {
+      get { return resourceNames; }
+    }
+
+    private HashSet<Resource> disabledParentResources;
+    public HashSet<Resource> DisabledParentResources {
+      get { return disabledParentResources; }
+    }
+
     private List<Plugin> onlinePlugins;
     public List<Plugin> OnlinePlugins {
@@ -93 +143 @@
 
       try {
+        projects = new ItemList<Project>();
+        resources = new ItemList<Resource>();
         jobs = new HiveItemCollection<RefreshableJob>();
-        var jobsLoaded = HiveServiceLocator.Instance.CallHiveService<IEnumerable<Job>>(s => s.GetJobs());
-
-        try {
-          foreach (var j in jobsLoaded) {
-            jobs.Add(new RefreshableJob(j));
-          }
-        } catch (NullReferenceException) {
-          // jobs was set to null during ClearHiveClient
-        }
+        projectNames = new Dictionary<Guid, string>();
+        resourceNames = new Dictionary<Guid, string>();
+
+        projectAncestors = new Dictionary<Guid, HashSet<Guid>>();
+        projectDescendants = new Dictionary<Guid, HashSet<Guid>>();
+        resourceAncestors = new Dictionary<Guid, HashSet<Guid>>();
+        resourceDescendants = new Dictionary<Guid, HashSet<Guid>>();
+
+        HiveServiceLocator.Instance.CallHiveService(service => {
+          service.GetProjects().ForEach(p => projects.Add(p));
+          service.GetSlaveGroups().ForEach(g => resources.Add(g));
+          service.GetSlaves().ForEach(s => resources.Add(s));
+          service.GetJobs().ForEach(p => jobs.Add(new RefreshableJob(p)));
+          projectNames = service.GetProjectNames();
+          resourceNames = service.GetResourceNames();
+        });
+
+        RefreshResourceGenealogy();
+        RefreshProjectGenealogy();
+        RefreshDisabledParentProjects();
+        RefreshDisabledParentResources();
+      }
+      catch {
+        jobs = null;
+        projects = null;
+        resources = null;
+        throw;
+      }
+      finally {
+        OnRefreshed();
+      }
+    }
+
+    public void RefreshProjectsAndResources() {
+      OnRefreshing();
+
+      try {
+        projects = new ItemList<Project>();
+        projectNames = new Dictionary<Guid, string>();
+        resources = new ItemList<Resource>();
+        resourceNames = new Dictionary<Guid, string>();
+
+        projectAncestors = new Dictionary<Guid, HashSet<Guid>>();
+        projectDescendants = new Dictionary<Guid, HashSet<Guid>>();
+        resourceAncestors = new Dictionary<Guid, HashSet<Guid>>();
+        resourceDescendants = new Dictionary<Guid, HashSet<Guid>>();
+
+        HiveServiceLocator.Instance.CallHiveService(service => {
+          service.GetProjects().ForEach(p => projects.Add(p));
+          service.GetSlaveGroups().ForEach(g => resources.Add(g));
+          service.GetSlaves().ForEach(s => resources.Add(s));
+          projectNames = service.GetProjectNames();
+          resourceNames = service.GetResourceNames();
+        });
+
+        RefreshResourceGenealogy();
+        RefreshProjectGenealogy();
+        RefreshDisabledParentProjects();
+        RefreshDisabledParentResources();
       } catch {
-        jobs = null;
+        projects = null;
+        resources = null;
         throw;
       } finally {
         OnRefreshed();
       }
+    }
+
+    public void RefreshAsync(Action<Exception> exceptionCallback) {
+      var call = new Func<Exception>(delegate() {
+        try {
+          Refresh();
+        }
+        catch (Exception ex) {
+          return ex;
+        }
+        return null;
+      });
+      call.BeginInvoke(delegate(IAsyncResult result) {
+        Exception ex = call.EndInvoke(result);
+        if (ex != null) exceptionCallback(ex);
+      }, null);
+    }
+
+    private void RefreshResourceGenealogy() {
+      resourceAncestors.Clear();
+      resourceDescendants.Clear();
+
+      // fetch resource ancestor set
+      HiveServiceLocator.Instance.CallHiveService(service => {
+        var ra = service.GetResourceGenealogy();
+        ra.Keys.ToList().ForEach(k => resourceAncestors.Add(k, new HashSet<Guid>()));
+        resourceAncestors.Keys.ToList().ForEach(k => resourceAncestors[k].UnionWith(ra[k]));
+      });
+
+      // build resource descendant set
+      resourceAncestors.Keys.ToList().ForEach(k => resourceDescendants.Add(k, new HashSet<Guid>()));
+      foreach (var ra in resourceAncestors) {
+        foreach(var ancestor in ra.Value) {
+          resourceDescendants[ancestor].Add(ra.Key);
+        }
+      }
+    }
+
+    private void RefreshProjectGenealogy() {
+      projectAncestors.Clear();
+      projectDescendants.Clear();
+
+      // fetch project ancestor list
+      HiveServiceLocator.Instance.CallHiveService(service => {
+        var pa = service.GetProjectGenealogy();
+        pa.Keys.ToList().ForEach(k => projectAncestors.Add(k, new HashSet<Guid>()));
+        projectAncestors.Keys.ToList().ForEach(k => projectAncestors[k].UnionWith(pa[k]));
+      });
+
+      // build project descendant list
+      projectAncestors.Keys.ToList().ForEach(k => projectDescendants.Add(k, new HashSet<Guid>()));
+      foreach(var pa in projectAncestors) {
+        foreach(var ancestor in pa.Value) {
+          projectDescendants[ancestor].Add(pa.Key);
+        }
+      }
+    }
+
+    private void RefreshDisabledParentProjects() {
+      disabledParentProjects = new HashSet<Project>();
+
+      foreach (var pid in projects
+        .Where(x => x.ParentProjectId.HasValue)
+        .SelectMany(x => projectAncestors[x.Id]).Distinct()
+        .Where(x => !projects.Select(y => y.Id).Contains(x))) {
+        var p = new Project();
+        p.Id = pid;
+        p.ParentProjectId = projectAncestors[pid].FirstOrDefault();
+        p.Name = projectNames[pid];
+        disabledParentProjects.Add(p);
+      }
+    }
+
+    private void RefreshDisabledParentResources() {
+      disabledParentResources = new HashSet<Resource>();
+
+      foreach (var rid in resources
+        .Where(x => x.ParentResourceId.HasValue)
+        .SelectMany(x => resourceAncestors[x.Id]).Distinct()
+        .Where(x => !resources.Select(y => y.Id).Contains(x))) {
+        var r = new SlaveGroup();
+        r.Id = rid;
+        r.ParentResourceId = resourceAncestors[rid].FirstOrDefault();
+        r.Name = resourceNames[rid];
+        disabledParentResources.Add(r);
+      }
+    }
+
+    public IEnumerable<Project> GetAvailableProjectAncestors(Guid id) {
+      if (projectAncestors.ContainsKey(id)) return projects.Where(x => projectAncestors[id].Contains(x.Id));
+      else return Enumerable.Empty<Project>();
+    }
+
+    public IEnumerable<Project> GetAvailableProjectDescendants(Guid id) {
+      if (projectDescendants.ContainsKey(id)) return projects.Where(x => projectDescendants[id].Contains(x.Id));
+      else return Enumerable.Empty<Project>();
+    }
+
+    public IEnumerable<Resource> GetAvailableResourceAncestors(Guid id) {
+      if (resourceAncestors.ContainsKey(id)) return resources.Where(x => resourceAncestors[id].Contains(x.Id));
+      else return Enumerable.Empty<Resource>();
+    }
+
+    public IEnumerable<Resource> GetAvailableResourceDescendants(Guid id) {
+      if (resourceDescendants.ContainsKey(id)) return resources.Where(x => resourceDescendants[id].Contains(x.Id));
+      else return Enumerable.Empty<Resource>();
+    }
+
+    public IEnumerable<Resource> GetAvailableResourcesForProject(Guid id) {
+      var assignedProjectResources = HiveServiceLocator.Instance.CallHiveService(s => s.GetAssignedResourcesForProject(id));
+      return resources.Where(x => assignedProjectResources.Select(y => y.ResourceId).Contains(x.Id));
+    }
+
+    public IEnumerable<Resource> GetDisabledResourceAncestors(IEnumerable<Resource> availableResources) {
+      var missingParentIds = availableResources
+        .Where(x => x.ParentResourceId.HasValue)
+        .SelectMany(x => resourceAncestors[x.Id]).Distinct()
+        .Where(x => !availableResources.Select(y => y.Id).Contains(x));
+
+      return resources.OfType<SlaveGroup>().Union(disabledParentResources).Where(x => missingParentIds.Contains(x.Id));
     }
     #endregion
@@ -116 +339 @@
       if (item.Id == Guid.Empty) {
         if (item is RefreshableJob) {
-          HiveClient.Instance.UploadJob((RefreshableJob)item, cancellationToken);
+          item.Id = HiveClient.Instance.UploadJob((RefreshableJob)item, cancellationToken);
         }
         if (item is JobPermission) {
@@ -126 +349 @@
           HiveServiceLocator.Instance.CallHiveService((s) => s.GrantPermission(hep.JobId, hep.GrantedUserId, hep.Permission));
         }
+        if (item is Project) {
+          item.Id = HiveServiceLocator.Instance.CallHiveService(s => s.AddProject((Project)item));
+        }
       } else {
-        if (item is Job)
-          HiveServiceLocator.Instance.CallHiveService(s => s.UpdateJob((Job)item));
+        if (item is Job) {
+          var job = (Job)item;
+          HiveServiceLocator.Instance.CallHiveService(s => s.UpdateJob(job, job.ResourceIds));
+        }
+        if (item is Project)
+          HiveServiceLocator.Instance.CallHiveService(s => s.UpdateProject((Project)item));
       }
     }
     public static void StoreAsync(Action<Exception> exceptionCallback, IHiveItem item, CancellationToken cancellationToken) {
-      var call = new Func<Exception>(delegate () {
+      var call = new Func<Exception>(delegate() {
         try {
           Store(item, cancellationToken);
-        } catch (Exception ex) {
+        }
+        catch (Exception ex) {
           return ex;
         }
         return null;
       });
-      call.BeginInvoke(delegate (IAsyncResult result) {
+      call.BeginInvoke(delegate(IAsyncResult result) {
         Exception ex = call.EndInvoke(result);
         if (ex != null) exceptionCallback(ex);
@@ -153 +384 @@
 
       if (item is Job)
-        HiveServiceLocator.Instance.CallHiveService(s => s.DeleteJob(item.Id));
+        HiveServiceLocator.Instance.CallHiveService(s => s.UpdateJobState(item.Id, JobState.StatisticsPending));
       if (item is RefreshableJob) {
         RefreshableJob job = (RefreshableJob)item;
@@ -159 +390 @@
           job.StopResultPolling();
         }
-        HiveServiceLocator.Instance.CallHiveService(s => s.DeleteJob(item.Id));
+        HiveServiceLocator.Instance.CallHiveService(s => s.UpdateJobState(item.Id, JobState.StatisticsPending));
       }
       if (item is JobPermission) {
@@ -227 +458 @@
     }
 
+    public static void UpdateJob(Action<Exception> exceptionCallback, RefreshableJob refreshableJob, CancellationToken cancellationToken) {
+      refreshableJob.IsProgressing = true;
+      refreshableJob.Progress.Status = "Saving Job...";
+      HiveClient.StoreAsync(
+        new Action<Exception>((Exception ex) => {
+          exceptionCallback(ex);
+        }), refreshableJob.Job, cancellationToken);
+      refreshableJob.IsProgressing = false;
+      refreshableJob.Progress.Finish();
+    }
+
+    public static void UpdateJob(RefreshableJob refreshableJob) {
+      refreshableJob.IsProgressing = true;
+
+      try {
+        refreshableJob.Progress.Start("Saving Job...");
+        HiveClient.StoreAsync(new Action<Exception>((Exception ex) => {
+          throw new Exception("Update failed.", ex);
+        }), refreshableJob.Job, new CancellationToken());
+      } finally {
+        refreshableJob.IsProgressing = false;
+        refreshableJob.Progress.Finish();
+      }
+    }
+
+
+
     #region Upload Job
     private Semaphore taskUploadSemaphore = new Semaphore(Settings.Default.MaxParallelUploads, Settings.Default.MaxParallelUploads);
     private static object jobCountLocker = new object();
     private static object pluginLocker = new object();
-    private void UploadJob(RefreshableJob refreshableJob, CancellationToken cancellationToken) {
+    private Guid UploadJob(RefreshableJob refreshableJob, CancellationToken cancellationToken) {
       try {
         refreshableJob.IsProgressing = true;
         refreshableJob.Progress.Start("Connecting to server...");
-        IEnumerable<string> resourceNames = ToResourceNameList(refreshableJob.Job.ResourceNames);
-        var resourceIds = new List<Guid>();
-        foreach (var resourceName in resourceNames) {
-          Guid resourceId = HiveServiceLocator.Instance.CallHiveService((s) => s.GetResourceId(resourceName));
-          if (resourceId == Guid.Empty) {
-            throw new ResourceNotFoundException(string.Format("Could not find the resource '{0}'", resourceName));
-          }
-          resourceIds.Add(resourceId);
-        }
 
         foreach (OptimizerHiveTask hiveJob in refreshableJob.HiveTasks.OfType<OptimizerHiveTask>()) {
@@ -251 +500 @@
         // upload Job
         refreshableJob.Progress.Status = "Uploading Job...";
-        refreshableJob.Job.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddJob(refreshableJob.Job));
+        refreshableJob.Job.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddJob(refreshableJob.Job, refreshableJob.Job.ResourceIds));
         refreshableJob.Job = HiveServiceLocator.Instance.CallHiveService((s) => s.GetJob(refreshableJob.Job.Id)); // update owner and permissions
         cancellationToken.ThrowIfCancellationRequested();
@@ -273 +522 @@
         foreach (HiveTask hiveTask in refreshableJob.HiveTasks) {
           var task = TS.Task.Factory.StartNew((hj) => {
-            UploadTaskWithChildren(refreshableJob.Progress, (HiveTask)hj, null, resourceIds, jobCount, totalJobCount, configFilePlugin.Id, refreshableJob.Job.Id, refreshableJob.Log, cancellationToken);
+            UploadTaskWithChildren(refreshableJob.Progress, (HiveTask)hj, null, jobCount, totalJobCount, configFilePlugin.Id, refreshableJob.Job.Id, refreshableJob.Log, cancellationToken);
           }, hiveTask);
           task.ContinueWith((x) => refreshableJob.Log.LogException(x.Exception), TaskContinuationOptions.OnlyOnFaulted);
@@ -279 +528 @@
         }
         TS.Task.WaitAll(tasks.ToArray());
-      } finally {
+      }
+      finally {
         refreshableJob.Job.Modified = false;
         refreshableJob.IsProgressing = false;
         refreshableJob.Progress.Finish();
       }
+      return (refreshableJob.Job != null) ? refreshableJob.Job.Id : Guid.Empty;
     }
 
@@ -317 +568 @@
     /// </summary>
     /// <param name="parentHiveTask">shall be null if its the root task</param>
-    private void UploadTaskWithChildren(IProgress progress, HiveTask hiveTask, HiveTask parentHiveTask, IEnumerable<Guid> groups, int[] taskCount, int totalJobCount, Guid configPluginId, Guid jobId, ILog log, CancellationToken cancellationToken) {
+    private void UploadTaskWithChildren(IProgress progress, HiveTask hiveTask, HiveTask parentHiveTask, int[] taskCount, int totalJobCount, Guid configPluginId, Guid jobId, ILog log, CancellationToken cancellationToken) {
       taskUploadSemaphore.WaitOne();
       bool semaphoreReleased = false;
@@ -356 +607 @@
               hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddChildTask(parentHiveTask.Task.Id, hiveTask.Task, taskData));
             } else {
-              hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddTask(hiveTask.Task, taskData, groups.ToList()));
+              hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddTask(hiveTask.Task, taskData));
             }
           }
@@ -371 +622 @@
           var task = TS.Task.Factory.StartNew((tuple) => {
             var arguments = (Tuple<HiveTask, HiveTask>)tuple;
-            UploadTaskWithChildren(progress, arguments.Item1, arguments.Item2, groups, taskCount, totalJobCount, configPluginId, jobId, log, cancellationToken);
+            UploadTaskWithChildren(progress, arguments.Item1, arguments.Item2, taskCount, totalJobCount, configPluginId, jobId, log, cancellationToken);
           }, new Tuple<HiveTask, HiveTask>(child, hiveTask));
           task.ContinueWith((x) => log.LogException(x.Exception), TaskContinuationOptions.OnlyOnFaulted);
@@ -426 +677 @@
         } else if (refreshableJob.IsPaused()) {
           refreshableJob.ExecutionState = Core.ExecutionState.Paused;
-        } else {
+        } else { 
           refreshableJob.ExecutionState = Core.ExecutionState.Started;
         }
         refreshableJob.OnLoaded();
-      } finally {
+      }
+      finally {
         refreshableJob.IsProgressing = false;
         refreshableJob.Progress.Finish();
@@ -467 +719 @@
       try {
         return PersistenceUtil.Deserialize<ItemTask>(taskData.Data);
-      } catch {
+      }
+      catch {
         return null;
       }
@@ -478 +731 @@
     public static void TryAndRepeat(Action action, int repetitions, string errorMessage, ILog log = null) {
       while (true) {
-        try { action(); return; } catch (Exception e) {
+        try { action(); return; }
+        catch (Exception e) {
           if (repetitions == 0) throw new HiveException(errorMessage, e);
           if (log != null) log.LogMessage(string.Format("{0}: {1} - will try again!", errorMessage, e.ToString()));
@@ -495 +749 @@
       });
     }
+
+    public string GetProjectAncestry(Guid projectId) {
+      if (projectId == null || projectId == Guid.Empty) return "";
+      var projects = projectAncestors[projectId].Reverse().ToList();
+      projects.Add(projectId);
+      return string.Join(" » ", projects.Select(x => ProjectNames[x]).ToArray());
+    }
+
+    public IEnumerable<Resource> GetAssignedResourcesForJob(Guid jobId) {
+      var assignedJobResource = HiveServiceLocator.Instance.CallHiveService(service => service.GetAssignedResourcesForJob(jobId));
+      return Resources.Where(x => assignedJobResource.Select(y => y.ResourceId).Contains(x.Id));
+    }
   }
 }