Changeset 17067

Timestamp:

07/04/19 15:20:26 (5 years ago)

Author:

mkommend

Message:

#2839: Merged 16622, 16878 into stable.

Location:

stable

Files:

• . (modified) (1 prop)
• HeuristicLab.Clients.Hive (modified) (1 prop)
• HeuristicLab.Clients.Hive.Administrator (modified) (1 prop)
• HeuristicLab.Clients.Hive.Administrator/3.3/Views/ProjectJobsView.cs (modified) (5 diffs)
• HeuristicLab.Clients.Hive.Slave (modified) (1 prop)
• HeuristicLab.Clients.Hive.Slave/3.3/Core.cs (modified) (3 diffs)
• HeuristicLab.Clients.Hive/3.3/HiveAdminClient.cs (modified) (16 diffs)
• HeuristicLab.Clients.Hive/3.3/ServiceClients/GenerateServiceClients.cmd (modified) (1 diff)
• HeuristicLab.Services.Hive (modified) (1 prop)
• HeuristicLab.Services.Hive.DataAccess (modified) (1 prop)
• HeuristicLab.Services.Hive.DataAccess/3.3/Daos/JobDao.cs (modified) (1 diff)
• HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs (modified) (2 diffs)

stable/HeuristicLab.Clients.Hive.Administrator/3.3/Views/ProjectJobsView.cs

@@ -26 +26 @@
 using System.Threading.Tasks;
 using System.Windows.Forms;
+using HeuristicLab.Clients.Hive.Views;
+using HeuristicLab.Core;
+using HeuristicLab.Core.Views;
+using HeuristicLab.Data;
 using HeuristicLab.MainForm;
 using HeuristicLab.MainForm.WindowsForms;
-using HeuristicLab.Core.Views;
-using HeuristicLab.Data;
-using HeuristicLab.Clients.Hive.Views;
-using HeuristicLab.Core;
 
 namespace HeuristicLab.Clients.Hive.Administrator.Views {
@@ -297 +297 @@
 
     private void RefreshJobs() {
-      HiveAdminClient.Instance.RefreshJobs();
+      HiveAdminClient.Instance.RefreshJobs(Content.Id);
       UpdateJobs();
       SetEnabledStateOfControls();
@@ -303 +303 @@
 
     private StringMatrix CreateValueMatrix() {
-      if (Content == null || Content.Id == Guid.Empty)
+      if (Content == null || Content.Id == Guid.Empty || !HiveAdminClient.Instance.Jobs.ContainsKey(Content.Id))
         return new StringMatrix();
 
@@ -352 +352 @@
 
             matrixView.DataGridView.AutoResizeColumns();
-            matrixView.DataGridView.Columns[0].MinimumWidth = 90;
-            matrixView.DataGridView.Columns[1].MinimumWidth = 108;
           }
         }
@@ -360 +358 @@
 
     private void RefreshJobsAsync() {
-      HiveAdminClient.Instance.RefreshJobs();
+      HiveAdminClient.Instance.RefreshJobs(Content.Id);
       UpdateJobs();
     }

stable/HeuristicLab.Clients.Hive.Slave/3.3/Core.cs

@@ -38 +38 @@
   /// </summary>
   public class Core : MarshalByRefObject {
+    private static readonly object locker = new object();
+
     private static HeartbeatManager heartbeatManager;
     public static HeartbeatManager HeartbeatManager {
@@ -240 +242 @@
         task = wcfService.GetTask(taskId);
         if (task == null) throw new TaskNotFoundException(taskId);
-        if (ConfigManager.Instance.GetFreeCores() < task.CoresNeeded) throw new OutOfCoresException();
-        if (ConfigManager.Instance.GetFreeMemory() < task.MemoryNeeded) throw new OutOfMemoryException();
-        SlaveStatusInfo.IncrementUsedCores(task.CoresNeeded); usedCores = task.CoresNeeded;
+        lock (locker) {
+          // the amount of used cores/memory could exceed the amount of available cores/memory
+          // if HandleCalculateTask is called simultaneously from within two different tasks
+          if (ConfigManager.Instance.GetFreeCores() < task.CoresNeeded) throw new OutOfCoresException();
+          if (ConfigManager.Instance.GetFreeMemory() < task.MemoryNeeded) throw new OutOfMemoryException();
+          SlaveStatusInfo.IncrementUsedCores(task.CoresNeeded); usedCores = task.CoresNeeded;
+        }
         TaskData taskData = wcfService.GetTaskData(taskId);
         if (taskData == null) throw new TaskDataNotFoundException(taskId);
@@ -400 +406 @@
 
     private void taskManager_TaskAborted(object sender, EventArgs<SlaveTask> e) {
+      var slaveTask = e.Value;
+      var task = wcfService.GetTask(slaveTask.TaskId);
+      wcfService.UpdateJobState(task.Id, TaskState.Aborted, null);
       SlaveStatusInfo.DecrementUsedCores(e.Value.CoresNeeded);
     }

stable/HeuristicLab.Clients.Hive/3.3/HiveAdminClient.cs

@@ -21 +21 @@
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
+using HeuristicLab.Clients.Access;
 using HeuristicLab.Common;
 using HeuristicLab.Core;
-using System.Collections.Generic;
-using System.Linq;
-using HeuristicLab.Clients.Access;
 
 namespace HeuristicLab.Clients.Hive {
@@ -77 +77 @@
         if (value != jobs)
           jobs = value;
-        }
+      }
     }
 
@@ -166 +166 @@
           if (projectIds.Any()) {
             service.GetAssignedResourcesForProjectsAdministration(projectIds)
-              .ForEach(a => projectResourceAssignments.Add(a));
-            projectIds.ForEach(p => jobs.Add(p, new HiveItemCollection<RefreshableJob>()));
+                   .ForEach(a => projectResourceAssignments.Add(a));
+
             var unsortedJobs = service.GetJobsByProjectIds(projectIds)
-              .OrderBy(x => x.DateCreated).ToList();
-
-            unsortedJobs.Where(j => j.State == JobState.DeletionPending).ToList().ForEach(j => jobs[j.ProjectId].Add(new RefreshableJob(j)));
-            unsortedJobs.Where(j => j.State == JobState.StatisticsPending).ToList().ForEach(j => jobs[j.ProjectId].Add(new RefreshableJob(j)));
-            unsortedJobs.Where(j => j.State == JobState.Online).ToList().ForEach(j => jobs[j.ProjectId].Add(new RefreshableJob(j)));
-
-            foreach (var job in jobs.SelectMany(x => x.Value))
-              LoadLightweightJob(job);
+                                      .OrderBy(x => x.DateCreated).ToList();
 
             projectNames = service.GetProjectNames();
@@ -187 +180 @@
         RefreshDisabledParentProjects();
         RefreshDisabledParentResources();
-      }
-      catch {
+      } catch {
         throw;
-      }
-      finally {
+      } finally {
         OnRefreshed();
       }
     }
-
-    //public void UpdateResourceGenealogy(IItemList<Resource> resources) {
-    //  resourceAncestors.Clear();
-    //  resourceDescendants.Clear();
-
-    //  foreach (var r in resources) {
-    //    resourceAncestors.Add(r.Id, new HashSet<Resource>());
-    //    resourceDescendants.Add(r.Id, new HashSet<Resource>());
-    //  }
-
-    //  foreach (var r in resources) {
-    //    var parentResourceId = r.ParentResourceId;
-    //    while (parentResourceId != null) {
-    //      var parent = resources.SingleOrDefault(x => x.Id == parentResourceId);
-    //      if (parent != null) {
-    //        resourceAncestors[r.Id].Add(parent);
-    //        resourceDescendants[parent.Id].Add(r);
-    //        parentResourceId = parent.ParentResourceId;
-    //      } else {
-    //        parentResourceId = null;
-    //      }
-    //    }
-    //  }
-    //}
-
-    //public void UpdateProjectGenealogy(IItemList<Project> projects) {
-    //  projectAncestors.Clear();
-    //  projectDescendants.Clear();
-
-    //  foreach (var p in projects) {
-    //    projectAncestors.Add(p.Id, new HashSet<Project>());
-    //    projectDescendants.Add(p.Id, new HashSet<Project>());
-    //  }
-
-    //  foreach (var p in projects) {
-    //    var parentProjectId = p.ParentProjectId;
-    //    while (parentProjectId != null) {
-    //      var parent = projects.SingleOrDefault(x => x.Id == parentProjectId);
-    //      if (parent != null) {
-    //        projectAncestors[p.Id].Add(parent);
-    //        projectDescendants[parent.Id].Add(p);
-    //        parentProjectId = parent.ParentProjectId;
-    //      } else {
-    //        parentProjectId = null;
-    //      }
-    //    }
-    //  }
-    //}
 
     private void UpdateResourceGenealogy() {
@@ -314 +257 @@
     }
 
-    public void RefreshJobs() {
-      var projectIds = new List<Guid>();
-      jobs = new Dictionary<Guid, HiveItemCollection<RefreshableJob>>();
-      tasks = new Dictionary<Guid, List<LightweightTask>>();
-
+    public void RefreshJobs(Guid projectId) {
       HiveServiceLocator.Instance.CallHiveService(service => {
-        service.GetProjectsForAdministration().ForEach(p => projectIds.Add(p.Id));
-        if(projectIds.Any()) {
-          projectIds.ForEach(p => jobs.Add(p, new HiveItemCollection<RefreshableJob>()));
-          var unsortedJobs = service.GetJobsByProjectIds(projectIds)
-            .OrderBy(x => x.DateCreated).ToList();
-          
+        var projectIds = new HashSet<Guid>(service.GetProjectsForAdministration().Select(x => x.Id));
+        if (projectIds.Contains(projectId)) {
+          jobs.Add(projectId, new HiveItemCollection<RefreshableJob>());
+
+          var unsortedJobs = service.GetJobsByProjectId(projectId)
+                                    .OrderBy(x => x.DateCreated).ToList();
+
           unsortedJobs.Where(j => j.State == JobState.DeletionPending).ToList().ForEach(j => jobs[j.ProjectId].Add(new RefreshableJob(j)));
           unsortedJobs.Where(j => j.State == JobState.StatisticsPending).ToList().ForEach(j => jobs[j.ProjectId].Add(new RefreshableJob(j)));
           unsortedJobs.Where(j => j.State == JobState.Online).ToList().ForEach(j => jobs[j.ProjectId].Add(new RefreshableJob(j)));
 
-          foreach(var job in jobs.SelectMany(x => x.Value))
+          foreach (var job in jobs.SelectMany(x => x.Value))
             LoadLightweightJob(job);
         }
@@ -344 +284 @@
         tasks[job.Id].AddRange(lightweightTasks);
       } else {
-        tasks.Add(job.Id, new List<LightweightTask>(lightweightTasks));        
+        tasks.Add(job.Id, new List<LightweightTask>(lightweightTasks));
       }
 
       if (lightweightTasks != null && lightweightTasks.Count > 0 && lightweightTasks.All(x => x.Id != Guid.Empty)) {
         if (lightweightTasks.All(x =>
-          x.State == TaskState.Finished 
-          || x.State == TaskState.Aborted 
+          x.State == TaskState.Finished
+          || x.State == TaskState.Aborted
           || x.State == TaskState.Failed)) {
           refreshableJob.ExecutionState = ExecutionState.Stopped;
@@ -366 +306 @@
           refreshableJob.ExecutionState = ExecutionState.Paused;
           refreshableJob.RefreshAutomatically = false;
-        } else if (lightweightTasks.Any(x => x.State == TaskState.Calculating 
-                                  || x.State == TaskState.Transferring 
+        } else if (lightweightTasks.Any(x => x.State == TaskState.Calculating
+                                  || x.State == TaskState.Transferring
                                   || x.State == TaskState.Waiting)) {
           refreshableJob.ExecutionState = ExecutionState.Started;
@@ -377 +317 @@
 
     public void SortJobs() {
-      for(int i = 0; i < jobs.Count; i++) {
+      for (int i = 0; i < jobs.Count; i++) {
         var projectId = jobs.Keys.ElementAt(i);
         var unsortedJobs = jobs.Values.ElementAt(i);
@@ -403 +343 @@
             service.GetDowntimesForResource(downtimeForResourceId).ForEach(d => downtimes.Add(d));
           });
-        }
-        catch {
+        } catch {
           throw;
-        }
-        finally {
+        } finally {
           OnRefreshed();
         }
@@ -521 +459 @@
 
     public IEnumerable<Project> GetAvailableProjectDescendants(Guid id) {
-      if(projectDescendants.ContainsKey(id)) return projects.Where(x => projectDescendants[id].Contains(x.Id));
+      if (projectDescendants.ContainsKey(id)) return projects.Where(x => projectDescendants[id].Contains(x.Id));
       else return Enumerable.Empty<Project>();
     }
@@ -545 +483 @@
 
     public bool CheckAccessToAdminAreaGranted() {
-      if(projects != null) {
+      if (projects != null) {
         return projects.Count > 0;
       } else {
@@ -561 +499 @@
       if (res.OwnerUserId == userId) {
         return true;
-      } else if(resourceAncestors.ContainsKey(res.Id)) {
+      } else if (resourceAncestors.ContainsKey(res.Id)) {
         return GetAvailableResourceAncestors(res.Id).Where(x => x.OwnerUserId == userId).Any();
       }
@@ -583 +521 @@
       if (pro == null || userId == Guid.Empty) return false;
 
-      if(projectAncestors.ContainsKey(pro.Id)) {
+      if (projectAncestors.ContainsKey(pro.Id)) {
         return GetAvailableProjectAncestors(pro.Id).Any(x => x.OwnerUserId == userId);
       }
@@ -613 +551 @@
         // ... if the user is no admin nor owner of the new parent or grand..grandparents
         changePossible = false;
-      } else if(parent != null && projectDescendants.ContainsKey(child.Id)) {
+      } else if (parent != null && projectDescendants.ContainsKey(child.Id)) {
         // ... if the new parent is among the moved project's descendants
         changePossible = !GetAvailableProjectDescendants(child.Id).Where(x => x.Id == parent.Id).Any();
@@ -630 +568 @@
       // ... or the new parent is a slave
       // ... or there is not parental change
-      if (child == null 
+      if (child == null
         || child == parent
         || (parent != null && parent.Id == Guid.Empty)
@@ -645 +583 @@
 
     public IEnumerable<Resource> GetAssignedResourcesForJob(Guid jobId) {
-      var assignedJobResource =  HiveServiceLocator.Instance.CallHiveService(service => service.GetAssignedResourcesForJob(jobId));
+      var assignedJobResource = HiveServiceLocator.Instance.CallHiveService(service => service.GetAssignedResourcesForJob(jobId));
       return Resources.Where(x => assignedJobResource.Select(y => y.ResourceId).Contains(x.Id));
     }

stable/HeuristicLab.Clients.Hive/3.3/ServiceClients/GenerateServiceClients.cmd

@@ -13 +13 @@
 IF "%GENERATECONFIG%"=="" SET GENERATECONFIG=y
 
-SET ARGS=http://%HOST%/Hive-3.3/HiveService.svc?wsdl ^
+SET ARGS=http://%HOST%/Hive-3.4/HiveService.svc?wsdl ^
 /out:HiveServiceClient ^
 /namespace:*,HeuristicLab.Clients.Hive ^

stable/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/JobDao.cs

@@ -88 +88 @@
         (from job in db.GetTable<Job>()
          where job.State == JobState.StatisticsPending
-         && (from task in db.GetTable<Task>()
-             where task.JobId == job.JobId
-             select task.State == TaskState.Finished
-              || task.State == TaskState.Aborted
-              || task.State == TaskState.Failed).All(x => x)
+               && (from task in db.GetTable<Task>()
+                   where task.JobId == job.JobId
+                   select task).All(x => x.State == TaskState.Finished
+                                      || x.State == TaskState.Aborted
+                                      || x.State == TaskState.Failed)
          select job));
     #endregion

stable/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

@@ -66 +66 @@
 
           pm.UseTransaction(() => {
-            UpdateFactTaskTable(pm);
             try {
-              pm.SubmitChanges();
+              UpdateFactTaskTable(pm);
               UpdateExistingDimJobs(pm);
+              FlagJobsForDeletion(pm);
               pm.SubmitChanges();
             }
@@ -81 +81 @@
           });
         }
-
-        pm.UseTransaction(() => {
-          FlagJobsForDeletion(pm);
-          pm.SubmitChanges();
-        });
       }
     }