Changeset 7287

Timestamp:

01/07/12 02:07:44 (13 years ago)

Author:

ascheibe

Message:

#1744

• added the new Hive engine. The engine can now be executed in the Hive. If child tasks are created it pauses and is transfered back to the server. If the child tasks are finished it is sent back to a slave.
• changed the server so that it reschedules paused parent tasks if their childs are finished as well as tasks where FinishWhenChildJobsFinished is set to false

Location:

branches/HiveHiveEngine

Files:

• HeuristicLab 3.3.sln (modified) (2 diffs)
• HeuristicLab.Clients.Hive/3.3/HiveClient.cs (modified) (4 diffs)
• HeuristicLab.HiveEngine (added)
• HeuristicLab.HiveEngine.Views (added)
• HeuristicLab.HiveEngine.Views/3.3 (added)
• HeuristicLab.HiveEngine.Views/3.3/HeuristicLab.HiveEngine.Views-3.3.csproj (added)
• HeuristicLab.HiveEngine.Views/3.3/HeuristicLab.snk (added)
• HeuristicLab.HiveEngine.Views/3.3/HiveEngineView.Designer.cs (added)
• HeuristicLab.HiveEngine.Views/3.3/HiveEngineView.cs (added)
• HeuristicLab.HiveEngine.Views/3.3/HiveEngineView.resx (added)
• HeuristicLab.HiveEngine.Views/3.3/Plugin.cs.frame (added)
• HeuristicLab.HiveEngine.Views/3.3/Properties (added)
• HeuristicLab.HiveEngine.Views/3.3/Properties/AssemblyInfo.cs.frame (added)
• HeuristicLab.HiveEngine/3.3 (added)
• HeuristicLab.HiveEngine/3.3/Exceptions (added)
• HeuristicLab.HiveEngine/3.3/Exceptions/HiveEngineException.cs (added)
• HeuristicLab.HiveEngine/3.3/Exceptions/ScopeMergeException.cs (added)
• HeuristicLab.HiveEngine/3.3/HeuristicLab.HiveEngine-3.3.csproj (added)
• HeuristicLab.HiveEngine/3.3/HeuristicLab.snk (added)
• HeuristicLab.HiveEngine/3.3/HiveEngine.cs (added)
• HeuristicLab.HiveEngine/3.3/Plugin.cs.frame (added)
• HeuristicLab.HiveEngine/3.3/Properties (added)
• HeuristicLab.HiveEngine/3.3/Properties/AssemblyInfo.cs.frame (added)
• HeuristicLab.Services.Hive/3.3/HiveDao.cs (modified) (2 diffs)
• HeuristicLab.Services.Hive/3.3/HiveService.cs (modified) (1 diff)

branches/HiveHiveEngine/HeuristicLab 3.3.sln

@@ -272 +272 @@
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.Algorithms.Benchmarks.Views-3.3", "HeuristicLab.Algorithms.Benchmarks.Views\3.3\HeuristicLab.Algorithms.Benchmarks.Views-3.3.csproj", "{3C906172-E044-4DF0-B4FD-AA21192D5D3E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.HiveEngine-3.3", "HeuristicLab.HiveEngine\3.3\HeuristicLab.HiveEngine-3.3.csproj", "{2C036542-5451-4A23-AFF6-87575C7BAFE7}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "HeuristicLab.HiveEngine.Views-3.3", "HeuristicLab.HiveEngine.Views\3.3\HeuristicLab.HiveEngine.Views-3.3.csproj", "{E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}"
 EndProject
 Global
@@ -1330 +1334 @@
     {3C906172-E044-4DF0-B4FD-AA21192D5D3E}.Release|x86.ActiveCfg = Release|x86
     {3C906172-E044-4DF0-B4FD-AA21192D5D3E}.Release|x86.Build.0 = Release|x86
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Debug|x64.ActiveCfg = Debug|x64
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Debug|x64.Build.0 = Debug|x64
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Debug|x86.ActiveCfg = Debug|x86
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Debug|x86.Build.0 = Debug|x86
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Release|Any CPU.Build.0 = Release|Any CPU
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Release|x64.ActiveCfg = Release|x64
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Release|x64.Build.0 = Release|x64
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Release|x86.ActiveCfg = Release|x86
+    {2C036542-5451-4A23-AFF6-87575C7BAFE7}.Release|x86.Build.0 = Release|x86
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Debug|x64.ActiveCfg = Debug|x64
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Debug|x64.Build.0 = Debug|x64
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Debug|x86.ActiveCfg = Debug|x86
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Debug|x86.Build.0 = Debug|x86
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Release|Any CPU.Build.0 = Release|Any CPU
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Release|x64.ActiveCfg = Release|x64
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Release|x64.Build.0 = Release|x64
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Release|x86.ActiveCfg = Release|x86
+    {E37F15E0-D0D6-4C6E-AD22-0DA514C943A3}.Release|x86.Build.0 = Release|x86
   EndGlobalSection
   GlobalSection(SolutionProperties) = preSolution

branches/HiveHiveEngine/HeuristicLab.Clients.Hive/3.3/HiveClient.cs

@@ -29 +29 @@
 using HeuristicLab.Common;
 using HeuristicLab.Core;
+using HeuristicLab.Optimization;
 using HeuristicLab.PluginInfrastructure;
 using TS = System.Threading.Tasks;
@@ -315 +316 @@
       finally {
         refreshableJob.IsProgressing = false;
+        refreshableJob.Progress.Finish();
       }
     }
@@ -364 +366 @@
           taskData = hiveTask.GetAsTaskData(true, out plugins);
         } else {
-          hiveTask.Task.IsParentTask = false;
+          if (hiveTask.ItemTask.Item is EngineAlgorithm && ((EngineAlgorithm)hiveTask.ItemTask.Item).Engine is IHiveEngine) {
+            hiveTask.Task.IsParentTask = true;
+            //TODO: this is a little hack so that this task does not get executed immediately because the taskdata will be uploaded twice
+            hiveTask.Task.State = TaskState.Aborted;
+          } else {
+            hiveTask.Task.IsParentTask = false;
+          }
+
           hiveTask.Task.FinishWhenChildJobsFinished = false;
           taskData = hiveTask.GetAsTaskData(false, out plugins);
@@ -389 +398 @@
             } else {
               hiveTask.Task.Id = HiveServiceLocator.Instance.CallHiveService((s) => s.AddTask(hiveTask.Task, taskData, groups.ToList()));
+
+              if (hiveTask.Task.State == TaskState.Aborted) {
+                //TODO: this is a very bad hack
+                if (hiveTask.ItemTask.Item is EngineAlgorithm && ((EngineAlgorithm)hiveTask.ItemTask.Item).Engine is IHiveEngine) {
+                  IHiveEngine he = ((EngineAlgorithm)hiveTask.ItemTask.Item).Engine as IHiveEngine;
+                  he.ParentTaskId = hiveTask.Task.Id;
+                  taskData = hiveTask.GetAsTaskData(false, out plugins);
+                  taskData.TaskId = hiveTask.Task.Id;
+
+                  Task t = HiveServiceLocator.Instance.CallHiveService((s) => s.GetTask(hiveTask.Task.Id));
+                  t.State = TaskState.Waiting;
+                  HiveServiceLocator.Instance.CallHiveService((s) => s.UpdateTaskData(t, taskData));
+                }
+              }
+
             }
           }

branches/HiveHiveEngine/HeuristicLab.Services.Hive/3.3/HiveDao.cs

@@ -103 +103 @@
                                || child.State == TaskState.Aborted
                                || child.State == TaskState.Failed).All(x => x)
+                    orderby ar.Task.Priority descending, db.Random()
+                    select DT.Convert.ToDto(ar.Task);
+        return count == 0 ? query.ToArray() : query.Take(count).ToArray();
+      }
+    }
+
+    /// <summary>
+    /// this is for the hive engine
+    /// </summary>    
+    private IEnumerable<DT.Task> GetPausedParentTasks(IEnumerable<Guid> resourceIds, int count) {
+      using (var db = CreateContext()) {
+        var query = from ar in db.AssignedResources
+                    where resourceIds.Contains(ar.ResourceId)
+                       && ar.Task.State == TaskState.Paused
+                       && ar.Task.IsParentTask
+                       && !ar.Task.FinishWhenChildJobsFinished
+                       && (from child in db.Tasks
+                           where child.ParentTaskId == ar.Task.TaskId
+                           select child).All(x => x.State == TaskState.Finished)
                        && (from child in db.Tasks // avoid returning WaitForChildTasks task where no child-task exist (yet)
                            where child.ParentTaskId == ar.Task.TaskId
@@ -115 +134 @@
       using (var db = CreateContext()) {
         var resourceIds = GetParentResources(slave.Id).Select(r => r.Id);
-        //Originally we checked here if there are parent tasks which should be calculated (with GetParentTasks(resourceIds, count, false);).
-        //Because there is at the moment no case where this makes sense (there don't exist parent tasks which need to be calculated), 
-        //we skip this step because it's wasted runtime
+
+        //check for freshly uploaded hive engine tasks
+        var waitingParentJobs = GetParentTasks(resourceIds, count, false);
+        if (count > 0 && waitingParentJobs.Count() >= count) return waitingParentJobs.Take(count).ToArray();
+
+        //check if there are hive engine tasks which have finished childs 
+        waitingParentJobs = GetPausedParentTasks(resourceIds, count);
+        if (count > 0 && waitingParentJobs.Count() >= count) return waitingParentJobs.Take(count).ToArray();
 
         var query = from ar in db.AssignedResources

branches/HiveHiveEngine/HeuristicLab.Services.Hive/3.3/HiveService.cs

@@ -71 +71 @@
         }
         dao.AddTaskData(taskData);
-        dao.UpdateTaskState(task.Id, DA.TaskState.Waiting, null, userManager.CurrentUserId, null);
+        //TODO: hack!! change this
+        if (task.State != TaskState.Aborted) {
+          dao.UpdateTaskState(task.Id, DA.TaskState.Waiting, null, userManager.CurrentUserId, null);
+        }
         return taskData.TaskId;
       }, false, true);