Changeset 5404

Timestamp:

02/01/11 15:51:11 (14 years ago)

Author:

cneumuel

Message:

#1233

• changed the workflow of aquireing a new job from server.
  • if a job is available for calculation, the slave receives the jobId already with the heartbeats. The job is then exclusively assigned to this slave.
• extended the metainfo for a slave by OperatingSystem and CpuArchitecture
• enhanced the way plugin-dependencies are discovered by using the types used by XmlGenerator. Now only mimimum amount of plugins are transferred.
• selection of waiting jobs now consideres assigned slave-group
• more unit tests for service
• added unit tests for experiment manager

Location:

branches/HeuristicLab.Hive-3.4/sources

Files:

• HeuristicLab.Clients.Hive.Slave.Tests/HeuristicLab.Clients.Hive.Slave.Tests-3.4.csproj (modified) (1 diff)
• HeuristicLab.Clients.Hive.Slave.Tests/SlaveTest.cs (modified) (4 diffs)
• HeuristicLab.Clients.Hive.Slave/3.4/ConfigManager.cs (modified) (5 diffs)
• HeuristicLab.Clients.Hive.Slave/3.4/Core.cs (modified) (1 diff)
• HeuristicLab.Clients.Hive.Slave/3.4/HeartbeatManager.cs (modified) (2 diffs)
• HeuristicLab.Clients.Hive.Slave/3.4/HeuristicLab.Clients.Hive.Slave-3.4.csproj (modified) (1 diff)
• HeuristicLab.Clients.Hive.Slave/3.4/WcfService.cs (modified) (1 diff)
• HeuristicLab.Clients.Hive.Tests (added)
• HeuristicLab.Clients.Hive.Tests/ExperimentManagerTests.cs (added)
• HeuristicLab.Clients.Hive.Tests/HeuristicLab.Clients.Hive.Tests-3.4.csproj (added)
• HeuristicLab.Clients.Hive.Tests/Mocks (added)
• HeuristicLab.Clients.Hive.Tests/Mocks/MockServiceLocator.cs (added)
• HeuristicLab.Clients.Hive.Tests/Properties (added)
• HeuristicLab.Clients.Hive.Tests/Properties/AssemblyInfo.cs (added)
• HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs (modified) (11 diffs)
• HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobClient.cs (modified) (3 diffs)
• HeuristicLab.Clients.Hive/3.4/PersistenceUtil.cs (modified) (1 diff)
• HeuristicLab.Clients.Hive/3.4/ServiceLocator.cs (modified) (1 diff)
• HeuristicLab.Hive 3.4.sln (modified) (3 diffs)
• HeuristicLab.Services.Hive.Common/3.4/DataTransfer/Slave.cs (modified) (2 diffs)
• HeuristicLab.Services.Hive.Common/3.4/MessageContainer.cs (modified) (1 diff)
• HeuristicLab.Services.Hive.Common/3.4/ServiceContracts/IHiveService.cs (modified) (1 diff)
• HeuristicLab.Services.Hive.DataAccess/3.4/Convert.cs (modified) (2 diffs)
• HeuristicLab.Services.Hive.DataAccess/3.4/Exceptions (added)
• HeuristicLab.Services.Hive.DataAccess/3.4/Exceptions/DaoException.cs (added)
• HeuristicLab.Services.Hive.DataAccess/3.4/HeuristicLab.Services.Hive.DataAccess-3.4.csproj (modified) (1 diff)
• HeuristicLab.Services.Hive.DataAccess/3.4/HiveDao.cs (modified) (3 diffs)
• HeuristicLab.Services.Hive.DataAccess/3.4/HiveDataContext.dbml (modified) (1 diff)
• HeuristicLab.Services.Hive.DataAccess/3.4/HiveDataContext.dbml.layout (modified) (5 diffs)
• HeuristicLab.Services.Hive.DataAccess/3.4/HiveDataContext.designer.cs (modified) (4 diffs)
• HeuristicLab.Services.Hive.DataAccess/3.4/Interfaces/IHiveDao.cs (modified) (1 diff)
• HeuristicLab.Services.Hive.DataAccess/3.4/Tools/prepareHiveDatabase.sql (modified) (1 diff)
• HeuristicLab.Services.Hive.Tests/DaoTests.cs (modified) (1 diff)
• HeuristicLab.Services.Hive.Tests/ServiceTests.cs (modified) (8 diffs)
• HeuristicLab.Services.Hive/3.4/HiveService.cs (modified) (1 diff)
• HeuristicLab.Services.Hive/3.4/LifecycleManager.cs (modified) (3 diffs)

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Tests/HeuristicLab.Clients.Hive.Slave.Tests-3.4.csproj

@@ -366 +366 @@
       <Name>HeuristicLab.Services.Hive.Common-3.4</Name>
     </ProjectReference>
+    <ProjectReference Include="..\HeuristicLab.Services.Hive\3.4\HeuristicLab.Services.Hive-3.4.csproj">
+      <Project>{CF9DA321-AC1B-4FD3-9EC3-67BC6B861BDE}</Project>
+      <Name>HeuristicLab.Services.Hive-3.4</Name>
+    </ProjectReference>
   </ItemGroup>
   <ItemGroup>

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave.Tests/SlaveTest.cs

@@ -53 +53 @@
       //get slave to fetch job
       List<MessageContainer> msg = new List<MessageContainer>();
-      msg.Add(new MessageContainer(MessageContainer.MessageType.AquireJob));
+      msg.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob));
       allMsgs.Add(msg);
 
@@ -99 +99 @@
       //get slave to fetch job
       List<MessageContainer> msg = new List<MessageContainer>();
-      msg.Add(new MessageContainer(MessageContainer.MessageType.AquireJob));
+      msg.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob));
       allMsgs.Add(msg);
 
@@ -135 +135 @@
       //get slave to fetch jobs
       List<MessageContainer> msg = new List<MessageContainer>();
-      msg.Add(new MessageContainer(MessageContainer.MessageType.AquireJob));
+      msg.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob));
       allMsgs.Add(msg);
 
@@ -142 +142 @@
 
       msg = new List<MessageContainer>();
-      msg.Add(new MessageContainer(MessageContainer.MessageType.AquireJob));
+      msg.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob));
       allMsgs.Add(msg);
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/ConfigManager.cs

@@ -26 +26 @@
 using HeuristicLab.Services.Hive.Common.DataTransfer;
 using HeuristicLab.Clients.Hive.Slave.Properties;
+using System.Management;
+using System.Diagnostics;
 
 
@@ -44 +46 @@
 
     public Core Core { get; set; }
-    private HeuristicLab.Services.Hive.Common.DataTransfer.Slave hardwareInfo;
-
+    private HeuristicLab.Services.Hive.Common.DataTransfer.Slave slave;
+    
     /// <summary>
     /// Constructor for the singleton, must recover Guid, Calendar, ...
     /// </summary>
     private ConfigManager() {
-      hardwareInfo = new HeuristicLab.Services.Hive.Common.DataTransfer.Slave();
-
-      if (Settings.Default.Guid == Guid.Empty) {
-        hardwareInfo.Id = Guid.NewGuid();
-        Settings.Default.Guid = hardwareInfo.Id;
-        Settings.Default.Save();
-      } else
-        hardwareInfo.Id = Settings.Default.Guid;
-
-      hardwareInfo.Cores = Environment.ProcessorCount;
-      hardwareInfo.Memory = 1024;
-      hardwareInfo.Name = Environment.MachineName;
+      slave = new HeuristicLab.Services.Hive.Common.DataTransfer.Slave();
+      slave.Id = GetUniqueMachineId();
+      slave.Name = Environment.MachineName;
+      slave.Cores = Environment.ProcessorCount;
+      slave.Memory = GetPhysicalMemory();
+      slave.CpuArchitecture = Environment.Is64BitOperatingSystem ? CpuArchitecture.x64 : CpuArchitecture.x86;
+      slave.OperatingSystem = Environment.OSVersion.VersionString;
     }
 
@@ -71 +68 @@
       //TODO: how to display connectedsince in gui?
       //hardwareInfo.Login = WcfService.Instance.ConnectedSince;
-      return hardwareInfo;
+      return slave;
     }
 
@@ -81 +78 @@
       //Todo: Locking
       StatusCommons st = new StatusCommons();
-      st.ClientGuid = hardwareInfo.Id;
+      st.ClientGuid = slave.Id;
 
       st.Status = WcfService.Instance.ConnState;
       st.ConnectedSince = WcfService.Instance.ConnectedSince;
 
-      st.TotalCores = hardwareInfo.Cores.HasValue ? hardwareInfo.Cores.Value : 0;
-      st.FreeCores = hardwareInfo.Cores.HasValue ? hardwareInfo.Cores.Value - GetUsedCores() : 0;
+      st.TotalCores = slave.Cores.HasValue ? slave.Cores.Value : 0;
+      st.FreeCores = slave.Cores.HasValue ? slave.Cores.Value - GetUsedCores() : 0;
 
       st.JobsAborted = SlaveStatusInfo.JobsAborted;
@@ -128 +125 @@
     }
 
+    public static Guid GetUniqueMachineId() {
+      // todo: instead of creating a new id, generate an ID from hardware IDs which is always the same for one machine
+      if (Settings.Default.Guid == Guid.Empty) {
+        Guid id = Guid.NewGuid();
+        Settings.Default.Guid = id;
+        Settings.Default.Save();
+        return id;
+      } else
+        return Settings.Default.Guid;
+    }
+
+    public static int GetPhysicalMemory() {
+      return 1024; // todo
+    }
+
+    public static int GetFreeMemory() {
+      PerformanceCounter counter = new PerformanceCounter("Memory", "Available Bytes", true);
+      int mb = (int)(counter.NextValue() / 1024 / 1024);
+      return mb;
+    }
   }
 }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/Core.cs

@@ -153 +153 @@
 
           //Pull a Job from the Server
-          case MessageContainer.MessageType.AquireJob:
-            Job myJob = wcfService.AquireJob();
+          case MessageContainer.MessageType.CalculateJob:
+            Job myJob = wcfService.GetJob(container.JobId);
             //TODO: handle in own thread!!
             JobData jobData = wcfService.GetJobData(myJob.Id);

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeartbeatManager.cs

@@ -93 +93 @@
                 SlaveId = info.Id, /*Settings.Default.Guid*/
                 FreeCores = info.Cores.HasValue ? info.Cores.Value - ConfigManager.Instance.GetUsedCores() : 0,
-                FreeMemory = GetFreeMemory(),
+                FreeMemory = ConfigManager.GetFreeMemory(),
                 JobProgress = ConfigManager.Instance.GetExecutionTimeOfAllJobs()
               };
@@ -129 +129 @@
     }
     #endregion
-
-    #region Helpers
-    private int GetFreeMemory() {
-      PerformanceCounter counter = new PerformanceCounter("Memory", "Available Bytes", true);
-      int mb = (int)(counter.NextValue() / 1024 / 1024);
-      return mb;
-    }
-    #endregion
   }
 }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/HeuristicLab.Clients.Hive.Slave-3.4.csproj

@@ -88 +88 @@
     <Reference Include="System" />
     <Reference Include="System.Core" />
+    <Reference Include="System.Management" />
     <Reference Include="System.Runtime.Serialization" />
     <Reference Include="System.ServiceModel" />

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/WcfService.cs

@@ -113 +113 @@
 
     /// <summary>
-    /// Aquire a Job from the Server, return the Job
-    /// </summary>
-    public Job AquireJob() {
-      using (Disposable<IHiveService> service = ServiceLocator.Instance.GetService()) {
-        try {
-          Job job = service.Obj.AquireJob(Settings.Default.Guid);
+    /// Get a Job from the Server
+    /// </summary>
+    public Job GetJob(Guid jobId) {
+      using (Disposable<IHiveService> service = ServiceLocator.Instance.GetService()) {
+        try {
+          Job job = service.Obj.GetJob(jobId);
           return job;
         }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveExperimentClient.cs

@@ -114 +114 @@
     }
 
-    private List<Guid> pluginsNeededIds;
-    public List<Guid> PluginsNeededIds {
-      get { return pluginsNeededIds; }
-      set { pluginsNeededIds = value; } 
+    private IEnumerable<Plugin> onlinePlugins;
+    public IEnumerable<Plugin> OnlinePlugins {
+      get { return onlinePlugins; }
+      set { onlinePlugins = value; } 
+    }
+
+    private List<Plugin> alreadyUploadedPlugins;
+    public List<Plugin> AlreadyUploadedPlugins {
+      get { return alreadyUploadedPlugins; }
+      set { alreadyUploadedPlugins = value; }
     }
 
@@ -247 +253 @@
 
           this.progress.Status = "Uploading plugins...";
-          this.PluginsNeededIds = GetPluginsNeededIds(this.useLocalPlugins);
+          this.OnlinePlugins = service.Obj.GetPlugins();
+          this.AlreadyUploadedPlugins = new List<Plugin>();
 
           this.progress.Status = "Uploading jobs...";
@@ -269 +276 @@
         IsProgressing = false;
       }
-    }
-
-    /// <summary>
-    /// Gets the Ids of all plugins needed for executing the job.
-    /// All loaded plugins are assumed to be necessary.
-    /// If a plugin with the same name and version is already online, it is used. Otherwise the local plugin is uploaded.
-    /// If useLocalPlugins is true, all local plugins are uploaded regardless of the existence of the same plugin online.
-    /// </summary>
-    public static List<Guid> GetPluginsNeededIds(bool useLocalPlugins) {
-      IEnumerable<IPluginDescription> localPlugins = ApplicationManager.Manager.Plugins;
-      List<Guid> pluginsNeededIds = new List<Guid>();
-
-      using (var service = ServiceLocator.Instance.GetService()) {
-        IEnumerable<Plugin> onlinePlugins = service.Obj.GetPlugins();
-
-        foreach (IPluginDescription localPlugin in localPlugins) {
-          Plugin found = onlinePlugins.Where(onlinePlugin => onlinePlugin.Name == localPlugin.Name && onlinePlugin.Version == localPlugin.Version).SingleOrDefault();
-          if (!useLocalPlugins && found != null) {
-            // plugin is available online; reuse
-            pluginsNeededIds.Add(found.Id);
-          } else {
-            // upload the plugin
-            Plugin p = new Plugin() { Name = localPlugin.Name, Version = localPlugin.Version, IsLocal = useLocalPlugins };
-            List<PluginData> pluginDatas = new List<PluginData>();
-
-            foreach (IPluginFile pf in localPlugin.Files) {
-              PluginData pluginData = new PluginData();
-
-              pluginData.Data = File.ReadAllBytes(pf.Name);
-              pluginDatas.Add(pluginData);
-            }
-            pluginsNeededIds.Add(service.Obj.AddPlugin(p, pluginDatas));
-          }
-        }
-      }
-      return pluginsNeededIds;
     }
 
@@ -319 +290 @@
       this.progress.Status = string.Format("Serializing job {0} of {1}", jobCount, totalJobCount);
       JobData jobData;
+      List<IPluginDescription> plugins;
       if (hiveJob.OptimizerJob.ComputeInParallel &&
         (hiveJob.OptimizerJob.Optimizer is Optimization.Experiment || hiveJob.OptimizerJob.Optimizer is Optimization.BatchRun)) {
         hiveJob.Job.JobState = JobState.WaitingForChildJobs;
         hiveJob.OptimizerJob.CollectChildJobs = false; // don't collect child-jobs on slaves
-        jobData = hiveJob.GetAsJobData(true);
+        jobData = hiveJob.GetAsJobData(true, out plugins);
       } else {
-        jobData = hiveJob.GetAsJobData(false);
-      }
-
-      hiveJob.Job.PluginsNeededIds = this.PluginsNeededIds;
+        jobData = hiveJob.GetAsJobData(false, out plugins);
+      }
+
+      hiveJob.Job.PluginsNeededIds = GetPluginDependencies(service, onlinePlugins, alreadyUploadedPlugins, plugins, useLocalPlugins);
 
       this.progress.Status = string.Format("Uploading job {0} of {1} ({2} kb)", jobCount, totalJobCount, jobData.Data.Count() / 1024);
@@ -347 +319 @@
       }
     }
-
+    
     /// <summary>
     /// Converts a string which can contain Ids separated by ';' to a enumerable
@@ -369 +341 @@
     #endregion
 
-    public void StartResultPolling() {
-      if (!jobResultPoller.IsPolling) {
-        jobResultPoller.Start();
-      } else {
-        throw new JobResultPollingException("Result polling already running");
-      }
-    }
-
-    public void StopResultPolling() {
-      if (jobResultPoller.IsPolling) {
-        jobResultPoller.Stop();
-      } else {
-        throw new JobResultPollingException("Result polling not running");
-      }
-    }
 
     #region HiveJob Events
@@ -483 +440 @@
 
     #region JobResultPoller Events
+
+    public void StartResultPolling() {
+      if (!jobResultPoller.IsPolling) {
+        jobResultPoller.Start();
+      } else {
+        throw new JobResultPollingException("Result polling already running");
+      }
+    }
+
+    public void StopResultPolling() {
+      if (jobResultPoller.IsPolling) {
+        jobResultPoller.Stop();
+      } else {
+        throw new JobResultPollingException("Result polling not running");
+      }
+    }
+
     private void RegisterResultPollingEvents() {
       jobResultPoller.ExceptionOccured += new EventHandler<EventArgs<Exception>>(jobResultPoller_ExceptionOccured);
@@ -497 +471 @@
       jobResultPoller.IsPollingChanged -= new EventHandler(jobResultPoller_IsPollingChanged);
     }
-    void jobResultPoller_IsPollingChanged(object sender, EventArgs e) {
+    private void jobResultPoller_IsPollingChanged(object sender, EventArgs e) {
       this.IsPollingResults = jobResultPoller.IsPolling;
     }
-    void jobResultPoller_PollingFinished(object sender, EventArgs e) {
+    private void jobResultPoller_PollingFinished(object sender, EventArgs e) {
       LogMessage("Polling results finished");
     }
-    void jobResultPoller_PollingStarted(object sender, EventArgs e) {
+    private void jobResultPoller_PollingStarted(object sender, EventArgs e) {
       LogMessage("Polling results started");
     }
-    void jobResultPoller_JobResultReceived(object sender, EventArgs<IEnumerable<LightweightJob>> e) {
+    private void jobResultPoller_JobResultReceived(object sender, EventArgs<IEnumerable<LightweightJob>> e) {
       foreach (LightweightJob lightweightJob in e.Value) {
         HiveJob hj = hiveJob.GetHiveJobByJobId(lightweightJob.Id);
@@ -543 +517 @@
     }
 
-    void jobResultPoller_ExceptionOccured(object sender, EventArgs<Exception> e) {
+    private void jobResultPoller_ExceptionOccured(object sender, EventArgs<Exception> e) {
       OnExceptionOccured(e.Value);
     }
@@ -578 +552 @@
     #endregion
 
+    #region Job Loading
     /// <summary>
     /// Downloads the root job from hive and sets the experiment, rootJob and rootJobItem
@@ -674 +649 @@
       }
     }
-    
+    #endregion 
+
+    #region Plugin Management
+    /// <summary>
+    /// Checks if plugins are available on Hive Server. If not they are uploaded. Ids are returned.
+    /// </summary>
+    /// <param name="service">An active service-proxy</param>
+    /// <param name="onlinePlugins">List of plugins which are available online</param>
+    /// <param name="alreadyUploadedPlugins">List of plugins which have been uploaded from this HiveExperiment</param>
+    /// <param name="neededPlugins">List of plugins which need to be uploaded</param>
+    /// <param name="useLocalPlugins">If true, the plugins which are already online are ignored. All local plugins are uploaded, but only once.</param>
+    /// <returns></returns>
+    private static List<Guid> GetPluginDependencies(IHiveService service, IEnumerable<Plugin> onlinePlugins, List<Plugin> alreadyUploadedPlugins, IEnumerable<IPluginDescription> neededPlugins, bool useLocalPlugins) {
+      var pluginIds = new List<Guid>();
+      foreach (var neededPlugin in neededPlugins) {
+        Plugin foundPlugin = alreadyUploadedPlugins.SingleOrDefault(p => p.Name == neededPlugin.Name && p.Version == neededPlugin.Version);
+        if (foundPlugin == null) {
+          foundPlugin = onlinePlugins.SingleOrDefault(p => p.Name == neededPlugin.Name && p.Version == neededPlugin.Version);
+          if (useLocalPlugins || foundPlugin == null) {
+            Plugin p = CreatePlugin(neededPlugin, useLocalPlugins);
+            List<PluginData> pd = CreatePluginDatas(neededPlugin);
+            p.Id = service.AddPlugin(p, pd);
+            alreadyUploadedPlugins.Add(p);
+          } else {
+            pluginIds.Add(foundPlugin.Id);
+          }
+        } else {
+          pluginIds.Add(foundPlugin.Id);
+        }
+      }
+      return pluginIds;
+    }
+
+    private static Plugin CreatePlugin(IPluginDescription plugin, bool useLocalPlugins) {
+      return new Plugin() { Name = plugin.Name, Version = plugin.Version, IsLocal = useLocalPlugins };
+    }
+
+    private static List<PluginData> CreatePluginDatas(IPluginDescription plugin) {
+      List<PluginData> pluginDatas = new List<PluginData>();
+
+      foreach (IPluginFile pf in plugin.Files) {
+        PluginData pluginData = new PluginData();
+
+        pluginData.Data = File.ReadAllBytes(pf.Name);
+        pluginData.FileName = Path.GetFileName(pf.Name);
+        pluginDatas.Add(pluginData);
+      }
+      return pluginDatas;
+    }
+
+    /// <summary>
+    /// Gets the Ids of all plugins needed for executing the job.
+    /// All loaded plugins are assumed to be necessary.
+    /// If a plugin with the same name and version is already online, it is used. Otherwise the local plugin is uploaded.
+    /// If useLocalPlugins is true, all local plugins are uploaded regardless of the existence of the same plugin online.
+    /// </summary>
+    //public static List<Guid> GetPluginsNeededIds(bool useLocalPlugins) {
+    //  IEnumerable<IPluginDescription> localPlugins = ApplicationManager.Manager.Plugins;
+    //  List<Guid> pluginsNeededIds = new List<Guid>();
+
+    //  using (var service = ServiceLocator.Instance.GetService()) {
+    //    IEnumerable<Plugin> onlinePlugins = service.Obj.GetPlugins();
+
+    //    foreach (IPluginDescription localPlugin in localPlugins) {
+    //      Plugin found = onlinePlugins.Where(onlinePlugin => onlinePlugin.Name == localPlugin.Name && onlinePlugin.Version == localPlugin.Version).SingleOrDefault();
+    //      if (!useLocalPlugins && found != null) {
+    //        // plugin is available online; reuse
+    //        pluginsNeededIds.Add(found.Id);
+    //      } else {
+    //        // upload the plugin
+    //        Plugin p = new Plugin() { Name = localPlugin.Name, Version = localPlugin.Version, IsLocal = useLocalPlugins };
+    //        List<PluginData> pluginDatas = new List<PluginData>();
+
+    //        foreach (IPluginFile pf in localPlugin.Files) {
+    //          PluginData pluginData = new PluginData();
+
+    //          pluginData.Data = File.ReadAllBytes(pf.Name);
+    //          pluginDatas.Add(pluginData);
+    //        }
+    //        pluginsNeededIds.Add(service.Obj.AddPlugin(p, pluginDatas));
+    //      }
+    //    }
+    //  }
+    //  return pluginsNeededIds;
+    //}
+    #endregion
   }
 }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ExperimentManager/HiveJobClient.cs

@@ -32 +32 @@
 using HeuristicLab.PluginInfrastructure;
 using HeuristicLab.Services.Hive.Common.DataTransfer;
+using HeuristicLab.PluginInfrastructure.Manager;
 
 namespace HeuristicLab.Clients.Hive {
@@ -425 +426 @@
     ///   if true the Child-Optimizers will not be serialized (if the job contains an Experiment)
     /// </param>
-    public JobData GetAsJobData(bool withoutChildOptimizers) {
+    public JobData GetAsJobData(bool withoutChildOptimizers, out List<IPluginDescription> plugins) {
+      plugins = new List<IPluginDescription>();
       if (this.optimizerJob == null || this.optimizerJob.Optimizer == null)
         return null;
 
+      IEnumerable<Type> usedTypes;
       byte[] jobByteArray;
       if (withoutChildOptimizers && this.OptimizerJob.Optimizer is Optimization.Experiment) {
         OptimizerJob clonedJob = (OptimizerJob)this.OptimizerJob.Clone(); // use a cloned job, so that the childHiveJob don't get confused
         clonedJob.OptimizerAsExperiment.Optimizers.Clear();
-        jobByteArray = PersistenceUtil.Serialize(clonedJob);
+        jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
       } else if (withoutChildOptimizers && this.OptimizerJob.Optimizer is Optimization.BatchRun) {
         OptimizerJob clonedJob = (OptimizerJob)this.OptimizerJob.Clone();
         clonedJob.OptimizerAsBatchRun.Optimizer = null;
-        jobByteArray = PersistenceUtil.Serialize(clonedJob);
+        jobByteArray = PersistenceUtil.Serialize(clonedJob, out usedTypes);
       } else if (this.OptimizerJob.Optimizer is IAlgorithm) {
         ((IAlgorithm)this.OptimizerJob.Optimizer).StoreAlgorithmInEachRun = false; // avoid storing the algorithm in runs to reduce size
-        jobByteArray = PersistenceUtil.Serialize(this.OptimizerJob);
+        jobByteArray = PersistenceUtil.Serialize(this.OptimizerJob, out usedTypes);
       } else {
-        jobByteArray = PersistenceUtil.Serialize(this.OptimizerJob);
+        jobByteArray = PersistenceUtil.Serialize(this.OptimizerJob, out usedTypes);
       }
 
@@ -450 +453 @@
       };
 
+      CollectDeclaringPlugins(plugins, usedTypes);
+
       return jobData;
+    }
+
+    private void CollectDeclaringPlugins(List<IPluginDescription> plugins, IEnumerable<Type> usedTypes) {
+      foreach (Type type in usedTypes) {
+        var plugin = ApplicationManager.Manager.GetDeclaringPlugin(type);
+        if (plugin != null && !plugins.Contains(plugin)) {
+          plugins.Add(plugin);
+          CollectPluginDependencies(plugins, plugin);
+        }
+      }
+    }
+
+    private void CollectPluginDependencies(List<IPluginDescription> plugins, IPluginDescription plugin) {
+      if (plugin == null) return;
+      foreach (var dependency in plugin.Dependencies) {
+        if (!plugins.Contains(dependency)) {
+          plugins.Add(dependency);
+          CollectPluginDependencies(plugins, dependency);
+        }
+      }
     }
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/PersistenceUtil.cs

@@ -22 +22 @@
 using System.IO;
 using HeuristicLab.Persistence.Default.Xml;
+using System.Collections.Generic;
+using HeuristicLab.PluginInfrastructure;
+using System;
+using HeuristicLab.Persistence.Core;
 
 namespace HeuristicLab.Clients.Hive {
   public static class PersistenceUtil {
+    public static byte[] Serialize(object obj, out IEnumerable<Type> types) {
+      MemoryStream memStream = new MemoryStream();
+      XmlGenerator.Serialize(obj, memStream, ConfigurationService.Instance.GetConfiguration(new XmlFormat()), false, out types);
+      byte[] jobByteArray = memStream.ToArray();
+      memStream.Dispose();
+      return jobByteArray;
+    }
+
     public static byte[] Serialize(object obj) {
       MemoryStream memStream = new MemoryStream();

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive/3.4/ServiceLocator.cs

@@ -39 +39 @@
 
     public Disposable<IHiveService> GetService() {
-      //TODO: move user config to app.config!
-      return ClientFactory.CreateClient<IHiveService>("wsHttpBinding_IHiveService", null, "ascheibe", "ascheibe$4");
+      return ClientFactory.CreateClient<IHiveService>("wsHttpBinding_IHiveService");
     }
   }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Hive 3.4.sln

@@ -31 +31 @@
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Services.Hive-3.4", "HeuristicLab.Services.Hive\3.4\HeuristicLab.Services.Hive-3.4.csproj", "{CF9DA321-AC1B-4FD3-9EC3-67BC6B861BDE}"
 EndProject
-Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Hive-3.4", "HeuristicLab.Services.Hive.Web\Hive-3.4\", "{0CA6706D-A569-45DE-A85C-4158891CC1BC}"
+Project("{E24C65DC-7377-472B-9ABA-BC803B73C61A}") = "Hive-3.4", "HeuristicLab.Services.Hive.Web\Hive-3.4", "{0CA6706D-A569-45DE-A85C-4158891CC1BC}"
   ProjectSection(WebsiteProperties) = preProject
     TargetFrameworkMoniker = ".NETFramework,Version%3Dv4.0"
@@ -64 +64 @@
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Clients.Hive.Slave.SlaveTrayIcon", "HeuristicLab.Clients.Hive.Slave.SlaveTrayIcon\HeuristicLab.Clients.Hive.Slave.SlaveTrayIcon.csproj", "{7C4B1DE4-FC9A-4448-BCF8-3CB3FA3CB8FA}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Clients.Hive.Tests-3.4", "HeuristicLab.Clients.Hive.Tests\HeuristicLab.Clients.Hive.Tests-3.4.csproj", "{8D40A723-139D-40E4-8BBA-4CB309A9E4B9}"
 EndProject
 Global
@@ -241 +243 @@
     {7C4B1DE4-FC9A-4448-BCF8-3CB3FA3CB8FA}.Release|x86.ActiveCfg = Release|x86
     {7C4B1DE4-FC9A-4448-BCF8-3CB3FA3CB8FA}.Release|x86.Build.0 = Release|x86
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Debug|x86.ActiveCfg = Debug|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Release|Any CPU.Build.0 = Release|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+    {8D40A723-139D-40E4-8BBA-4CB309A9E4B9}.Release|x86.ActiveCfg = Release|Any CPU
   EndGlobalSection
   GlobalSection(SolutionProperties) = preSolution

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Common/3.4/DataTransfer/Slave.cs

@@ -25 +25 @@
 
 namespace HeuristicLab.Services.Hive.Common.DataTransfer {
+  public enum CpuArchitecture {
+    x86, x64
+  }
+
   [DataContract]
   [Serializable]
@@ -35 +39 @@
     public int? CpuSpeed { get; set; } // MHz
     [DataMember]
+    public CpuArchitecture CpuArchitecture { get; set; }
+    [DataMember]
     public int? Memory { get; set; } // MB
     [DataMember]
     public int? FreeMemory { get; set; } // MB
+    [DataMember]
+    public string OperatingSystem { get; set; }
     [DataMember]
     public SlaveState SlaveState { get; set; }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Common/3.4/MessageContainer.cs

@@ -35 +35 @@
     public enum MessageType {
       // *** commands from hive server ***
-      AquireJob, // slave should aquire a new job (formerly FetchJob)
+      CalculateJob, // slave should calculate a job. the job is already assigned to the slave
       StopJob,   // slave should stop the job and submit results
       AbortJob,  // slave should shut the job down immediately without submitting results

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Common/3.4/ServiceContracts/IHiveService.cs

@@ -40 +40 @@
     [OperationContract]
     void DeleteChildJobs(Guid parentJobId);
-
-    [OperationContract] // new method: appropriate job is choosen and set to 'calculating'. the slave is responsible for requesting the jobData. Server should wait some timeout until he redistributes the job
-    Job AquireJob(Guid slaveId);
 
     [OperationContract]

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/Convert.cs

@@ -143 +143 @@
     public static DT.Slave ToDto(Slave source) {
       if (source == null) return null;
-      return new DT.Slave { Id = source.ResourceId, ParentResourceId = source.ParentResourceId, Cores = source.Cores, CpuSpeed = source.CpuSpeed, FreeCores = source.FreeCores, FreeMemory = source.FreeMemory, IsAllowedToCalculate = source.IsAllowedToCalculate, Memory = source.Memory, Name = source.Name, SlaveState = source.SlaveState };
+      return new DT.Slave { Id = source.ResourceId, ParentResourceId = source.ParentResourceId, Cores = source.Cores, CpuSpeed = source.CpuSpeed, FreeCores = source.FreeCores, FreeMemory = source.FreeMemory, IsAllowedToCalculate = source.IsAllowedToCalculate, Memory = source.Memory, Name = source.Name, SlaveState = source.SlaveState, CpuArchitecture = source.CpuArchitecture, OperatingSystem = source.OperatingSystem };
     }
     public static Slave ToEntity(DT.Slave source) {
@@ -152 +152 @@
     public static void ToEntity(DT.Slave source, Slave target) {
       if ((source != null) && (target != null)) {
-        target.ResourceId = source.Id; target.ParentResourceId = source.ParentResourceId; target.Cores = source.Cores; target.CpuSpeed = source.CpuSpeed; target.FreeCores = source.FreeCores; target.FreeMemory = source.FreeMemory; target.IsAllowedToCalculate = source.IsAllowedToCalculate; target.Memory = source.Memory; target.Name = source.Name; target.SlaveState = source.SlaveState;
+        target.ResourceId = source.Id; target.ParentResourceId = source.ParentResourceId; target.Cores = source.Cores; target.CpuSpeed = source.CpuSpeed; target.FreeCores = source.FreeCores; target.FreeMemory = source.FreeMemory; target.IsAllowedToCalculate = source.IsAllowedToCalculate; target.Memory = source.Memory; target.Name = source.Name; target.SlaveState = source.SlaveState; target.CpuArchitecture = source.CpuArchitecture; target.OperatingSystem = source.OperatingSystem;
       }
     }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/HeuristicLab.Services.Hive.DataAccess-3.4.csproj

@@ -103 +103 @@
     <Compile Include="Convert.cs" />
     <None Include="HeuristicLabServicesHiveDataAccessPlugin.cs.frame" />
+    <Compile Include="Exceptions\DaoException.cs" />
     <Compile Include="HeuristicLabServicesHiveDataAccessPlugin.cs" />
     <Compile Include="HiveDao.cs" />

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/HiveDao.cs

@@ -82 +82 @@
     }
 
-    public IEnumerable<DT.Job> GetWaitingParentJobs(Guid slaveId) {
-      using (var db = CreateContext()) {
-        // todo: slaveId is unused!
+    public IEnumerable<DT.Job> GetWaitingParentJobs(IEnumerable<Guid> resourceIds, int count) {
+      using (var db = CreateContext()) {
         var query = from ar in db.AssignedResources
-                    where ar.Job.JobState == JobState.WaitingForChildJobs &&
-                      (from child in db.Jobs
-                       where child.ParentJobId == ar.Job.JobId
-                       select child.JobState == JobState.Finished).All(x => x) &&
-                      (from child in db.Jobs // avoid returning WaitForChildJobs jobs where no child-jobs exist (yet)
-                       where child.ParentJobId == ar.Job.JobId
-                       select child).Count() > 0
+                    where resourceIds.Contains(ar.ResourceId)
+                       && ar.Job.JobState == JobState.WaitingForChildJobs
+                       && (from child in db.Jobs
+                           where child.ParentJobId == ar.Job.JobId
+                           select child.JobState == JobState.Finished).All(x => x)
+                       && (from child in db.Jobs // avoid returning WaitForChildJobs jobs where no child-jobs exist (yet)
+                           where child.ParentJobId == ar.Job.JobId
+                           select child).Count() > 0
                     orderby ar.Job.Priority descending
                     select Convert.ToDto(ar.Job);
-        return query.ToArray();
-      }
-    }
-
-    public IEnumerable<DT.Job> GetWaitingJobs(DT.Slave slave) {
-      using (var db = CreateContext()) {
-        var query = from j in db.Jobs
-                    where j.JobState == JobState.Waiting && j.CoresNeeded <= slave.FreeCores && j.MemoryNeeded <= slave.FreeMemory
-                    orderby j.Priority descending
-                    select Convert.ToDto(j);
-        var waitingJobs = query.ToArray();
-        var waitingParentJobs = GetWaitingParentJobs(slave.Id);
+        return count == 0 ? query.ToArray() : query.Take(count).ToArray();
+      }
+    }
+
+    public IEnumerable<DT.Job> GetWaitingJobs(DT.Slave slave, int count) {
+      using (var db = CreateContext()) {
+        var resourceIds = GetParentResources(slave.Id).Select(r => r.Id);
+        var waitingParentJobs = GetWaitingParentJobs(resourceIds, count);
+        if (count > 0 && waitingParentJobs.Count() >= count) return waitingParentJobs.Take(count).ToArray();
+
+        var query = from ar in db.AssignedResources
+                    where resourceIds.Contains(ar.ResourceId)
+                       && ar.Job.JobState == JobState.Waiting
+                       && ar.Job.CoresNeeded <= slave.FreeCores
+                       && ar.Job.MemoryNeeded <= slave.FreeMemory
+                    orderby ar.Job.Priority descending
+                    select Convert.ToDto(ar.Job);
+        var waitingJobs = (count == 0 ? query : query.Take(count)).ToArray();
         return waitingJobs.Union(waitingParentJobs).OrderByDescending(x => x.Priority);
       }
@@ -350 +356 @@
       using (var db = CreateContext()) {
         var entity = db.Resources.OfType<SlaveGroup>().FirstOrDefault(x => x.ResourceId == id);
-        if (entity != null) db.Resources.DeleteOnSubmit(entity);
+        if (entity != null) {
+          if (db.Resources.Where(r => r.ParentResourceId == id).Count() > 0) {
+            throw new DaoException("Cannot delete SlaveGroup as long as there are Slaves in the group");
+          }
+          db.Resources.DeleteOnSubmit(entity);
+        }
         db.SubmitChanges();
       }
@@ -408 +419 @@
         return job.AssignedResources.Select(x => Convert.ToDto(x.Resource)).ToArray();
       }
+    }
+
+    /// <summary>
+    /// Returns all parent resources of a resource (the given resource is also added)
+    /// </summary>
+    private IEnumerable<DT.Resource> GetParentResources(Guid resourceId) {
+      using (var db = CreateContext()) {
+        var resources = new List<Resource>();
+        CollectParentResources(resources, db.Resources.Where(r => r.ResourceId == resourceId).Single());
+        return resources.Select(r => Convert.ToDto(r)).ToArray();
+      }
+    }
+
+    private void CollectParentResources(List<Resource> resources, Resource resource) {
+      if (resource == null) return;
+      resources.Add(resource);
+      CollectParentResources(resources, resource.ParentResource);
     }
     #endregion

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/HiveDataContext.dbml

@@ -49 +49 @@
         <Column Name="FreeMemory" Type="System.Int32" DbType="Int" CanBeNull="true" />
         <Column Name="IsAllowedToCalculate" Type="System.Boolean" DbType="Bit" CanBeNull="false" />
+        <Column Name="CpuArchitecture" Type="global::HeuristicLab.Services.Hive.Common.DataTransfer.CpuArchitecture" DbType="VarChar(3)" CanBeNull="false" />
+        <Column Name="OperatingSystem" Type="System.String" DbType="VarChar(MAX)" CanBeNull="false" />
         <Association Name="Slave_Job" Member="Jobs" ThisKey="ResourceId" OtherKey="SlaveId" Type="Job" />
       </Type>

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/HiveDataContext.dbml.layout

@@ -45 +45 @@
       </nestedChildShapes>
     </classShape>
-    <classShape Id="26f4edfa-91dd-4941-a058-359f89e567a8" absoluteBounds="8.875, 1, 2, 2.3478011067708335">
+    <classShape Id="26f4edfa-91dd-4941-a058-359f89e567a8" absoluteBounds="8.875, 1, 2, 2.7324039713541666">
       <DataClassMoniker Name="/HiveDataContext/Slave" />
       <nestedChildShapes>
-        <elementListCompartment Id="1e61f36b-08dc-4df7-8594-c9dcd95c0791" absoluteBounds="8.89, 1.46, 1.9700000000000002, 1.7878011067708333" name="DataPropertiesCompartment" titleTextColor="Black" itemTextColor="Black" />
+        <elementListCompartment Id="1e61f36b-08dc-4df7-8594-c9dcd95c0791" absoluteBounds="8.89, 1.46, 1.9700000000000002, 2.1724039713541665" name="DataPropertiesCompartment" titleTextColor="Black" itemTextColor="Black" />
       </nestedChildShapes>
     </classShape>
@@ -69 +69 @@
       </nodes>
     </inheritanceConnector>
-    <associationConnector edgePoints="[(12.25 : 2.57859537760417); (12.25 : 4.44314697265625); (10.875 : 4.44314697265625)]" fixedFrom="NotFixed" fixedTo="NotFixed">
+    <associationConnector edgePoints="[(11.9843735 : 2.57859537760417); (11.9843735 : 4.44314697265625); (10.875 : 4.44314697265625)]" fixedFrom="NotFixed" fixedTo="NotFixed">
       <AssociationMoniker Name="/HiveDataContext/Resource/Resource_AssignedResource" />
       <nodes>
@@ -97 +97 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(8.875 : 2.17390055338542); (8.5 : 2.17390055338542)]" fixedFrom="NotFixed" fixedTo="NotFixed">
+    <associationConnector edgePoints="[(8.875 : 2.36620198567708); (8.5 : 2.36620198567708)]" fixedFrom="NotFixed" fixedTo="NotFixed">
       <AssociationMoniker Name="/HiveDataContext/Slave/Slave_Job" />
       <nodes>
@@ -130 +130 @@
       </nestedChildShapes>
     </classShape>
-    <associationConnector edgePoints="[(11 : 6.28929768880208); (11.625 : 6.28929768880208)]" fixedFrom="Algorithm" fixedTo="Algorithm">
+    <associationConnector edgePoints="[(11 : 6.28929768880208); (11.625 : 6.28929768880208)]" fixedFrom="NotFixed" fixedTo="NotFixed">
       <AssociationMoniker Name="/HiveDataContext/Plugin/Plugin_PluginData" />
       <nodes>
@@ -144 +144 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(9 : 6.19314697265625); (8.5 : 6.19314697265625)]" fixedFrom="Algorithm" fixedTo="Algorithm">
+    <associationConnector edgePoints="[(9 : 6.19314697265625); (8.5 : 6.19314697265625)]" fixedFrom="NotFixed" fixedTo="NotFixed">
       <AssociationMoniker Name="/HiveDataContext/Plugin/Plugin_RequiredPlugin" />
       <nodes>

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/HiveDataContext.designer.cs

@@ -3 +3 @@
 // <auto-generated>
 //     This code was generated by a tool.
-//     Runtime Version:4.0.30319.1
+//     Runtime Version:4.0.30319.208
 //
 //     Changes to this file may cause incorrect behavior and will be lost if
@@ -1060 +1060 @@
     private bool _IsAllowedToCalculate;
     
+    private global::HeuristicLab.Services.Hive.Common.DataTransfer.CpuArchitecture _CpuArchitecture;
+    
+    private string _OperatingSystem;
+    
     private EntitySet<Job> _Jobs;
     
@@ -1082 +1086 @@
     partial void OnIsAllowedToCalculateChanging(bool value);
     partial void OnIsAllowedToCalculateChanged();
+    partial void OnCpuArchitectureChanging(global::HeuristicLab.Services.Hive.Common.DataTransfer.CpuArchitecture value);
+    partial void OnCpuArchitectureChanged();
+    partial void OnOperatingSystemChanging(string value);
+    partial void OnOperatingSystemChanged();
     #endregion
     
@@ -1246 +1254 @@
           this.SendPropertyChanged("IsAllowedToCalculate");
           this.OnIsAllowedToCalculateChanged();
+        }
+      }
+    }
+    
+    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_CpuArchitecture", DbType="VarChar(3)", CanBeNull=false)]
+    public global::HeuristicLab.Services.Hive.Common.DataTransfer.CpuArchitecture CpuArchitecture
+    {
+      get
+      {
+        return this._CpuArchitecture;
+      }
+      set
+      {
+        if ((this._CpuArchitecture != value))
+        {
+          this.OnCpuArchitectureChanging(value);
+          this.SendPropertyChanging();
+          this._CpuArchitecture = value;
+          this.SendPropertyChanged("CpuArchitecture");
+          this.OnCpuArchitectureChanged();
+        }
+      }
+    }
+    
+    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_OperatingSystem", DbType="VarChar(MAX)", CanBeNull=false)]
+    public string OperatingSystem
+    {
+      get
+      {
+        return this._OperatingSystem;
+      }
+      set
+      {
+        if ((this._OperatingSystem != value))
+        {
+          this.OnOperatingSystemChanging(value);
+          this.SendPropertyChanging();
+          this._OperatingSystem = value;
+          this.SendPropertyChanged("OperatingSystem");
+          this.OnOperatingSystemChanged();
         }
       }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/Interfaces/IHiveDao.cs

@@ -15 +15 @@
     void UpdateJob(DT.Job dto);
     void DeleteJob(Guid id);
-    IEnumerable<DT.Job> GetWaitingJobs(DT.Slave slave);
+    IEnumerable<DT.Job> GetWaitingJobs(DT.Slave slave, int count);
     #endregion
 

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.DataAccess/3.4/Tools/prepareHiveDatabase.sql

@@ -40 +40 @@
 ALTER TABLE dbo.Job ALTER COLUMN JobId ADD ROWGUIDCOL;
 ALTER TABLE dbo.Job WITH NOCHECK ADD CONSTRAINT [DF_Job_JobId] DEFAULT (newid()) FOR JobId;
+
+--ALTER TABLE [dbo].[Job]  DROP  CONSTRAINT [Slave_Job]
+--ALTER TABLE [dbo].[Job]  WITH CHECK ADD  CONSTRAINT [Slave_Job] FOREIGN KEY([ResourceId])
+--REFERENCES [dbo].[Resource] ([ResourceId])
+--ON UPDATE CASCADE
+--ON DELETE SET NULL
+--GO
 
 ALTER TABLE dbo.Plugin ALTER COLUMN PluginId ADD ROWGUIDCOL;

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests/DaoTests.cs

@@ -76 +76 @@
 
       DT.Slave slave = new DT.Slave() {
-        Name = "Test"
+        Name = "Test",
+        OperatingSystem = Environment.OSVersion.VersionString,
+        Cores = 2,
+        Memory = 1024
       };
       slave.Id= dao.AddSlave(slave);

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive.Tests/ServiceTests.cs

@@ -28 +28 @@
 using HeuristicLab.Services.Hive.Common.ServiceContracts;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
+using System.Threading;
+using HeuristicLab.Hive;
+using HeuristicLab.Services.Hive.Common;
 
 namespace HeuristicLab.Services.Hive.Tests {
-  using System.Threading;
-  using HeuristicLab.Hive;
-  using HeuristicLab.Services.Hive.Common;
+
   using DT = HeuristicLab.Services.Hive.Common.DataTransfer;
+  using System.Diagnostics;
 
   [TestClass]
@@ -66 +68 @@
         Data = PersistenceUtil.Serialize(new MockJob(500, true))
       };
+
+      DT.Plugin plugin1 = new DT.Plugin();
+      plugin1.Name = "Tests.MyPlugin";
+      plugin1.Version = new Version("1.0.0.0");
+      plugin1.UserId = Guid.Empty;
+      plugin1.IsLocal = true;
+      plugin1.DateCreated = DateTime.Now;
+
+      DT.PluginData pluginData1 = new DT.PluginData();
+      pluginData1.PluginId = plugin1.Id;
+      pluginData1.FileName = "Tests.MyPlugin-1.0.dll";
+      pluginData1.Data = new byte[] { 0, 1, 2, 3, 4, 5 };
+
+      job.PluginsNeededIds.Add(plugin1.Id);
 
       job.Id = service.AddJob(job, jobData, null);
@@ -77 +93 @@
       Assert.AreEqual(JobState.Waiting, jobLoaded.JobState);
       Assert.AreEqual(ServiceLocator.Instance.AuthorizationManager.UserId, job.UserId);
+      Assert.IsTrue(job.PluginsNeededIds.SequenceEqual(jobLoaded.PluginsNeededIds));
 
       DT.JobData jobDataLoaded = service.GetJobData(job.Id);
@@ -101 +118 @@
     public void TestHeartbeats() {
       var service = GetLocalService();
+      // check if group already exists and delete
+      var existingSlaveGroup = service.GetSlaveGroups().SingleOrDefault(g => g.Name == "TestGroup");
+      if (existingSlaveGroup != null) {
+        var slavesToDelete = service.GetSlaves().Where(s => s.ParentResourceId == existingSlaveGroup.Id);
+        foreach (var slave in slavesToDelete) service.DeleteSlave(slave.Id);
+        service.DeleteSlaveGroup(existingSlaveGroup.Id);
+      }
+
       Guid groupId = service.AddSlaveGroup(new SlaveGroup() { Name = "TestGroup", Description = "Used for unit tests" });
 
-      for (int i = 0; i < 2; i++) {
-        Job job = new Job() {
-          CoresNeeded = 1, MemoryNeeded = 0
-        };
-        JobData jobData = new JobData() { Data = PersistenceUtil.Serialize(new MockJob(500, false)) };
-        job.Id = service.AddJob(job, jobData, null);
-        jobs.Add(job);
-      }
-
+      // create slaves
       var slaves = new List<DT.Slave>();
       for (int i = 0; i < 1; i++) {
@@ -124 +141 @@
           FreeMemory = 3000
         };
+        // check if slave with this name already exists and delete
+        var existingSlave = service.GetSlaves().Where(s => s.Name == slave.Name).SingleOrDefault();
+        if (existingSlave != null) service.DeleteSlave(existingSlave.Id);
+
         slave.Id = service.AddSlave(slave);
         service.AddResourceToGroup(groupId, slave.Id);
@@ -129 +150 @@
       }
 
+      // create jobs with different group, they should not be assigned
+      existingSlaveGroup = service.GetSlaveGroups().SingleOrDefault(g => g.Name == "DummyGroup");
+      if (existingSlaveGroup != null) service.DeleteSlaveGroup(existingSlaveGroup.Id);
+
+      Guid dummyGroupId = service.AddSlaveGroup(new SlaveGroup() { Name = "DummyGroup", Description = "Used for unit tests; jobs from this group shall not be calculated" });
+      // create dummy jobs
+      var dummyJobs = new List<Job>();
+      for (int i = 0; i < 2; i++) {
+        Job job = new Job() {
+          CoresNeeded = 1, MemoryNeeded = 0
+        };
+        JobData jobData = new JobData() { Data = PersistenceUtil.Serialize(new MockJob(500, false)) };
+        job.Id = service.AddJob(job, jobData, new List<Guid> { dummyGroupId });
+        dummyJobs.Add(job);
+      }
+
+      // create jobs
+      for (int i = 0; i < 2; i++) {
+        Job job = new Job() {
+          CoresNeeded = 1, MemoryNeeded = 0
+        };
+        JobData jobData = new JobData() { Data = PersistenceUtil.Serialize(new MockJob(500, false)) };
+        job.Id = service.AddJob(job, jobData, new List<Guid> { groupId });
+        jobs.Add(job);
+      }
+
+      // send heartbeats
       foreach (var slave in slaves) {
         new Thread(new ParameterizedThreadStart(RunSlaveThread)).Start(slave);
       }
 
-      // send heartbeats
       IEnumerable<LightweightJob> lightweightJobs;
       do {
@@ -146 +193 @@
       }
 
-      // delete group
+      // delete groups
       service.DeleteSlaveGroup(groupId);
+      service.DeleteSlaveGroup(dummyGroupId);
 
       // delete jobs
       foreach (var job in jobs) {
+        service.DeleteJob(job.Id);
+      }
+
+      // delete dummy jobs
+      foreach (var job in dummyJobs) {
         service.DeleteJob(job.Id);
       }
@@ -161 +214 @@
         int freeCores = slave.Cores.Value;
 
-        var messages = service.Heartbeat(new Heartbeat() { SlaveId = slave.Id, FreeMemory = 2423, FreeCores = freeCores, JobProgress = new Dictionary<Guid, TimeSpan>() });
-        if (messages.Count == 0)
-          return; // no more jobs
-
-        Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.AbortJob).Count() == 0);
-        Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.SayHello).Count() == 0);
-        Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.PauseJob).Count() == 0);
-
-        if (messages.Where(x => x.Message == MessageContainer.MessageType.AquireJob).Count() > 0) {
-          Guid jobId = messages.Where(x => x.Message == MessageContainer.MessageType.AquireJob).SingleOrDefault().JobId;
-          service.AquireJob(jobId);
-          Job job = service.GetJob(jobId);
-          JobData jobData = service.GetJobData(jobId);
-          IJob deserializedJob = PersistenceUtil.Deserialize<IJob>(jobData.Data);
-          deserializedJob.Start();
-          job.JobState = JobState.Finished;
-          jobs.Where(x => x.Id == jobId).Single().JobState = JobState.Finished;
-          jobData.Data = PersistenceUtil.Serialize(deserializedJob);
-          service.UpdateJob(job, jobData);
+        for (int i = 0; i < 10; i++) {
+
+          var messages = service.Heartbeat(new Heartbeat() { SlaveId = slave.Id, FreeMemory = 2423, FreeCores = freeCores, JobProgress = new Dictionary<Guid, TimeSpan>() });
+          if (messages.Count == 0) {
+            Debug.WriteLine("No job available");
+            return; // no more jobs
+          }
+
+          Debug.WriteLine("Messages: {0}", string.Join(", ", messages.Select(m => m.Message)));
+
+          Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.AbortJob).Count() == 0);
+          Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.SayHello).Count() == 0);
+          Assert.IsTrue(messages.Where(x => x.Message == MessageContainer.MessageType.PauseJob).Count() == 0);
+
+          var calculateJobMessage = messages.Where(x => x.Message == MessageContainer.MessageType.CalculateJob).SingleOrDefault();
+          if (calculateJobMessage != null) {
+            if (!jobs.Select(j => j.Id).Contains(calculateJobMessage.JobId))
+              Assert.Fail("Got job which was not assigned to the slavegroup");
+
+            Debug.WriteLine("Job available, calculating");
+            Job job = service.GetJob(calculateJobMessage.JobId);
+
+            JobData jobData = service.GetJobData(job.Id);
+            IJob deserializedJob = PersistenceUtil.Deserialize<IJob>(jobData.Data);
+            deserializedJob.Start();
+            job.JobState = JobState.Finished;
+            jobs.Where(x => x.Id == job.Id).Single().JobState = JobState.Finished;
+            jobData.Data = PersistenceUtil.Serialize(deserializedJob);
+            service.UpdateJob(job, jobData);
+            Debug.WriteLine("finished calculating");
+          }
         }
       }

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/HiveService.cs

@@ -100 +100 @@
       }
     }
-
-    public Job AquireJob(Guid slaveId) {
-      using (trans.OpenTransaction()) {
-        var slave = dao.GetSlave(slaveId);
-        var availableJobs = dao.GetWaitingJobs(slave);
-        var job = availableJobs.FirstOrDefault();
-
-        if (job != null) {
-          job.SlaveId = slaveId;
-          job.JobState = JobState.Calculating;
-        }
-        return job;
-      }
-    }
-
+    
     public PluginData GetConfigurationFile() {
       using (trans.OpenTransaction()) {

branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Services.Hive/3.4/LifecycleManager.cs

@@ -94 +94 @@
     /// if not -> set them offline and check if they where calculating a job
     /// </summary>
-    void timer_Tick(object sender, EventArgs e) {
+    private void timer_Tick(object sender, EventArgs e) {
       lock (locker) {
         using (trans.OpenTransaction()) {
@@ -146 +146 @@
 
         if (this.IsAllowedToSendJobs() && slave.IsAllowedToCalculate && heartbeat.FreeCores > 0) {
-          var availableJobs = dao.GetWaitingJobs(slave);
+          var availableJobs = dao.GetWaitingJobs(slave, 1);
           if (availableJobs.Count() > 0) {
-            actions.Add(new MessageContainer(MessageContainer.MessageType.AquireJob));
+            var job = availableJobs.First();
+            actions.Add(new MessageContainer(MessageContainer.MessageType.CalculateJob, job.Id));
+            AssignJob(slave, job);
           }
         }
@@ -164 +166 @@
       }
       return actions;
+    }
+
+    private void AssignJob(Slave slave, Job job) {
+      job.SlaveId = slave.Id;
+      job.JobState = JobState.Calculating; // Todo: Maybe use State = Transferring (?)
+      job.DateCalculated = DateTime.Now; // Todo: use statelog instead
+      dao.UpdateJob(job);
+      dao.UpdateSlave(slave);
     }