Changeset 12484 for branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3

Timestamp:

06/19/15 18:06:19 (9 years ago)

Author:

dglaser

Message:

#2388: Updated Hive, DataAccess and WebApp

HeuristicLab.Services.Hive.DataAccess:

• Updated database statistics schema

HeuristicLab.Services.Hive:

• Fixed event flag in HiveJanitor Service
• Improved UpdateTaskFactsTable in the HiveStatisticsGenerator

HeuristicLab.Services.WebApp:

• Updated Statistics DataController to match the new statistics schema

Location:

branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3

Files:

• HiveJanitor.cs (modified) (1 diff)
• HiveStatisticsGenerator.cs (modified) (9 diffs)

branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveJanitor.cs

@@ -49 +49 @@
     public HiveJanitor() {
       stop = false;
-      cleanupWaitHandle = new AutoResetEvent(true);
-      generateStatisticsWaitHandle = new AutoResetEvent(true);
+      cleanupWaitHandle = new AutoResetEvent(false);
+      generateStatisticsWaitHandle = new AutoResetEvent(false);
     }
 

branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

@@ -38 +38 @@
           UpdateDimUserTable(pm);
           UpdateDimJobTable(pm);
-          UpdateDimClients(pm);
+          UpdateDimClientsTable(pm);
           pm.SubmitChanges();
         });
@@ -47 +47 @@
             pm.SubmitChanges();
             UpdateFactClientInfoTable(newTime, pm);
-            UpdateTaskFacts(newTime, pm);
+            UpdateTaskFactsTable(newTime, pm);
             try {
               pm.SubmitChanges();
@@ -67 +67 @@
       var now = DateTime.Now;
       var timeEntry = new DimTime {
-        Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
+        Time = now,
+        Minute = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
         Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
         Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
@@ -73 +74 @@
         Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
       };
-      // using SaveOrAttach instead of save because the table can only be updated once every minute
-      // (pk restriction) save would cause a duplicate key exception if the method is accidentally 
-      // called more than once every minute (e.g. restarting the service within a minute)
-      return dimTimeDao.SaveOrAttach(timeEntry);
+      return dimTimeDao.Save(timeEntry);
     }
 
@@ -118 +116 @@
     }
 
-    private void UpdateDimClients(PersistenceManager pm) {
+    private void UpdateDimClientsTable(PersistenceManager pm) {
       var dimClientDao = pm.DimClientDao;
       var slaveDao = pm.SlaveDao;
@@ -208 +206 @@
     }
 
-    private void UpdateTaskFacts(DimTime newTime, PersistenceManager pm) {
+    private void UpdateTaskFactsTable(DimTime newTime, PersistenceManager pm) {
       var factTaskDao = pm.FactTaskDao;
       var taskDao = pm.TaskDao;
@@ -217 +215 @@
       var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
         x.TaskId,
-        x.DimTimeStart,
         x.LastClientId
       });
@@ -238 +235 @@
           MemoryRequired = task.MemoryNeeded,
           State = task.State,
-          StateLogs = stateLogs,
-          DimTimeStart = lastFact != null ? lastFact.DimTimeStart : newTime,
+          StateLogs = stateLogs.OrderBy(x => x.DateTime),
           LastClientId = client != null
                          ? client.Id : lastFact != null
                          ? lastFact.LastClientId : (Guid?)null
         };
-
-      var endStates = new[] { TaskState.Finished, TaskState.Failed, TaskState.Aborted };
       factTaskDao.Save(
         from x in newTasks.ToList()
-        let stateLogsLinkedList = new LinkedList<StateLog>(x.StateLogs.OrderBy(y => y.DateTime))
-        let lastStateLog = stateLogsLinkedList.OrderByDescending(y => y.DateTime).FirstOrDefault(sl => sl.Exception != null)
+        let taskData = CalculateFactTaskData(x.StateLogs)
         select new FactTask {
           TaskId = x.TaskId,
           JobId = x.JobId,
-          DimTimeStart = x.DimTimeStart,
-          DimTimeEnd = endStates.Contains(x.State) ? newTime : null,
+          StartTime = taskData.StartTime,
+          EndTime = taskData.EndTime,
           LastClientId = x.LastClientId,
           Priority = x.Priority,
           CoresRequired = x.CoresRequired,
           MemoryRequired = x.MemoryRequired,
-          NumCalculationRuns = stateLogsLinkedList.CountCalculationRuns(),
-          NumRetries = stateLogsLinkedList.CountRetries(),
-          TotalWaitingTime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Waiting),
-          TotalRuntime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(y => y.State == TaskState.Transferring)),
-          TotalTransferTime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Transferring),
+          NumCalculationRuns = taskData.CalculationRuns,
+          NumRetries = taskData.Retries,
+          WaitingTime = taskData.WaitingTime,
+          CalculatingTime = taskData.CalculatingTime,
+          TransferTime = taskData.TransferTime,
           TaskState = x.State,
-          Exception = lastStateLog == null ? string.Empty : lastStateLog.Exception
+          Exception = taskData.Exception,
+          InitialWaitingTime = taskData.InitialWaitingTime
         });
       factTaskDao.Delete(notFinishedFactTasks.Select(x => x.TaskId));
@@ -280 +274 @@
       }
     }
-  }
-
-  public static class StateLogLinkedListExtensions {
-    public static int CountCalculationRuns(this LinkedList<StateLog> stateLogs) {
-      return stateLogs.EnumerateNodes()
-                      .Count(sl => sl.Value.State == TaskState.Calculating && sl.NextIs(nsl => nsl.State == TaskState.Transferring));
-    }
-
-    public static int CountRetries(this LinkedList<StateLog> stateLogs) {
-      return stateLogs.EnumerateNodes()
-                      .Count(sl => sl.Value.State == TaskState.Calculating && sl.Next != null && sl.NextIs(nsl => nsl.State != TaskState.Transferring));
-    }
-
-    public static double SumTotalTimeWhere(this LinkedList<StateLog> stateLogs, Predicate<LinkedListNode<StateLog>> predicate) {
-      return stateLogs.EnumerateNodes()
-                      .Where(stateLog => predicate(stateLog))
-                      .Sum(stateLog => stateLog.Next != null ? (stateLog.Next.Value.DateTime - stateLog.Value.DateTime).TotalMinutes : 0.0);
-    }
-  }
-
-  public static class LinkedListExtensions {
-    public static IEnumerable<LinkedListNode<T>> EnumerateNodes<T>(this LinkedList<T> list) {
-      var node = list.First;
-      while (node != null) {
-        yield return node;
-        node = node.Next;
+
+    private class FactTaskData {
+      public int CalculationRuns { get; set; }
+      public int Retries { get; set; }
+      public double CalculatingTime { get; set; }
+      public double WaitingTime { get; set; }
+      public double TransferTime { get; set; }
+      public double InitialWaitingTime { get; set; }
+      public string Exception { get; set; }
+      public DateTime? StartTime { get; set; }
+      public DateTime? EndTime { get; set; }
+    }
+
+    private FactTaskData CalculateFactTaskData(IEnumerable<StateLog> stateLogs) {
+      var factTaskData = new FactTaskData();
+      var enumerator = stateLogs.GetEnumerator();
+      if (enumerator.MoveNext()) {
+        StateLog current = enumerator.Current, first = current, prev = null;
+        while (current != null) {
+          var next = enumerator.MoveNext() ? enumerator.Current : null;
+          int timeSpanInSeconds;
+          if (next != null) {
+            timeSpanInSeconds = (int)(next.DateTime - current.DateTime).TotalSeconds;
+          } else {
+            timeSpanInSeconds = (int)(DateTime.Now - current.DateTime).TotalSeconds;
+            factTaskData.Exception = current.Exception;
+          }
+          switch (current.State) {
+            case TaskState.Calculating:
+              factTaskData.CalculatingTime += timeSpanInSeconds;
+              factTaskData.CalculationRuns++;
+              if (factTaskData.CalculationRuns == 1) {
+                factTaskData.StartTime = current.DateTime;
+                factTaskData.InitialWaitingTime = (int)(current.DateTime - first.DateTime).TotalSeconds;
+              }
+              if (prev != null && prev.State != TaskState.Transferring) {
+                factTaskData.Retries++;
+              }
+              break;
+
+            case TaskState.Waiting:
+              factTaskData.WaitingTime += timeSpanInSeconds;
+              break;
+
+            case TaskState.Transferring:
+              factTaskData.TransferTime += timeSpanInSeconds;
+              break;
+
+            case TaskState.Finished:
+            case TaskState.Failed:
+            case TaskState.Aborted:
+              factTaskData.EndTime = current.DateTime;
+              break;
+          }
+          prev = current;
+          current = next;
+        }
       }
-    }
-
-    public static bool NextIs<T>(this LinkedListNode<T> node, Predicate<T> predicate) {
-      return node.Next != null && predicate(node.Next.Value);
+      return factTaskData;
     }
   }