Changeset 9571 for branches/HiveStatistics

Timestamp:

06/03/13 15:18:12 (11 years ago)

Author:

pfleck

Message:

#2063:
Calculating total waiting-, transfer- and calculating time for task statistics.

Location:

branches/HiveStatistics/sources

Files:

• HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml (modified) (1 diff)
• HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs (modified) (3 diffs)
• HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql (modified) (1 diff)
• HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs (modified) (7 diffs)

branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml

@@ -230 +230 @@
       <Column Name="TotalTransferTime" Type="System.Double" DbType="Float NOT NULL" CanBeNull="false" />
       <Column Name="NumCalculationRuns" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
-      <Column Name="NumFails" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
+      <Column Name="NumRetries" Storage="_NumFails" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
       <Column Name="CoresRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
       <Column Name="MemoryRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />

branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs

@@ -4908 +4908 @@
     partial void OnNumCalculationRunsChanging(int value);
     partial void OnNumCalculationRunsChanged();
-    partial void OnNumFailsChanging(int value);
-    partial void OnNumFailsChanged();
+    partial void OnNumRetriesChanging(int value);
+    partial void OnNumRetriesChanged();
     partial void OnCoresRequiredChanging(int value);
     partial void OnCoresRequiredChanged();
@@ -5040 +5040 @@
     
     [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_NumFails", DbType="Int NOT NULL")]
-    public int NumFails
+    public int NumRetries
     {
       get
@@ -5050 +5050 @@
         if ((this._NumFails != value))
         {
-          this.OnNumFailsChanging(value);
+          this.OnNumRetriesChanging(value);
           this.SendPropertyChanging();
           this._NumFails = value;
-          this.SendPropertyChanged("NumFails");
-          this.OnNumFailsChanged();
+          this.SendPropertyChanged("NumRetries");
+          this.OnNumRetriesChanged();
         }
       }

branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql

@@ -258 +258 @@
     [TotalTransferTime]  FLOAT (53)       NOT NULL,
     [NumCalculationRuns] INT              NOT NULL,
-    [NumFails]           INT              NOT NULL,
+    [NumRetries]         INT              NOT NULL,
     [CoresRequired]      INT              NOT NULL,
     [MemoryRequired]     INT              NOT NULL,

branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

@@ -1 +1 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Transactions;
@@ -128 +129 @@
         from s in db.Resources.OfType<Slave>()
         where !db.DimClients.Select(x => x.ResourceId).Contains(s.ResourceId)
-          || modifiedClients.Select(x => x.Slave.ResourceId).Contains(s.ResourceId)
+              || modifiedClients.Select(x => x.Slave.ResourceId).Contains(s.ResourceId)
         select new {
           Slave = s,
@@ -222 +223 @@
 
     private void UpdateTaskFacts(DimTime newTime, HiveDataContext db) {
+      var time = newTime.Time;
+
       var lastFacts =
         from fact in db.FactTasks
@@ -231 +234 @@
         join lastFact in lastFacts on task.TaskId equals lastFact.TaskId into lastFactsPerTask
         from lastFact in lastFactsPerTask.DefaultIfEmpty()
-        let newStateLogs = task.StateLogs.Where(stateLog => (lastFact == null || stateLog.DateTime > lastFact.EndTime) && stateLog.DateTime < newTime.Time)
+        let newStateLogs = task.StateLogs.Where(stateLog => (lastFact == null || stateLog.DateTime > lastFact.EndTime) && stateLog.DateTime < time)
+        let lastStateLog = lastFact == null ? null : (task.StateLogs.Where(stateLog => stateLog.DateTime < lastFact.EndTime).OrderByDescending(stateLog => stateLog.DateTime).First())
         let lastSlaveId = newStateLogs.OrderByDescending(stateLog => stateLog.DateTime).First(stateLog => stateLog.SlaveId != null).SlaveId
         join client in db.DimClients.Where(client => client.ExpirationTime == null) on lastSlaveId equals client.ResourceId into clientPerTask
         from client in clientPerTask.DefaultIfEmpty()
+        where newStateLogs.Any()
         select new {
           Task = task,
@@ -240 +245 @@
           LastFact = lastFact,
           StateLogs = newStateLogs.OrderBy(stateLog => stateLog.DateTime),
+          LastStateLog = lastStateLog,
           LastClientId = client != null ? client.Id : default(Guid?)
         };
@@ -245 +251 @@
       var newTaskFacts =
         from t in lastFactAndStateLogsAndLastClientPerTask.ToList()
+        let stateLogsLinkedList = t.LastStateLog == null ? new LinkedList<StateLog>(t.StateLogs) : new LinkedList<StateLog>(t.StateLogs).AddFirst(t.LastStateLog).List
         select new FactTask {
           TaskId = t.Task.TaskId,
@@ -256 +263 @@
           TaskSize = t.LastFact != null ? t.LastFact.TaskSize : t.TaskDataSize,
           ResultSize = t.Task.State == TaskState.Finished ? t.TaskDataSize : default(long?),
-          NumCalculationRuns = (t.LastFact != null ? t.LastFact.NumCalculationRuns : 0) + t.StateLogs.Count(x => x.State == TaskState.Calculating),
-          NumFails = (t.LastFact != null ? t.LastFact.NumFails : 0) + t.StateLogs.Count(x => x.State == TaskState.Failed || !string.IsNullOrEmpty(x.Exception)),
-          TotalWaitingTime = 0, //ToDo
-          TotalRuntime = 0, //ToDo
-          TotalTransferTime = 0 //ToDo
+          NumCalculationRuns =
+            (t.LastFact != null ? t.LastFact.NumCalculationRuns : 0) +
+            t.StateLogs.Count(x => x.State == TaskState.Calculating),
+          NumRetries =
+            (t.LastFact != null ? t.LastFact.NumRetries : 0) +
+            stateLogsLinkedList.EnumerateNodes().Count(sl => sl.Value.State == TaskState.Calculating && sl.Next != null && !sl.NextIs(nsl => nsl.State == TaskState.Transferring)),       //TotalWaitingTime = SumTotalWaitingTime(t.LastFact, stateLogsLinkedList),
+          TotalWaitingTime = SumTotalTime(t.LastFact, fact => fact.TotalWaitingTime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Waiting),
+          TotalRuntime = SumTotalTime(t.LastFact, fact => fact.TotalRuntime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(x => x.State == TaskState.Transferring)),
+          TotalTransferTime = SumTotalTime(t.LastFact, fact => fact.TotalTransferTime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Transferring)
         };
 
       db.FactTasks.InsertAllOnSubmit(newTaskFacts);
+    }
+
+    private double SumTotalTime(FactTask lastFact, Func<FactTask, double> lastValue, LinkedList<StateLog> stateLogs, Predicate<LinkedListNode<StateLog>> predicate) {
+      double sum = lastFact != null ? lastValue(lastFact) : 0.0;
+
+      sum += stateLogs.EnumerateNodes()
+                      .Where(sl => predicate(sl))
+                      .Sum(stateLog => stateLog.Next != null ? (stateLog.Next.Value.DateTime - stateLog.Value.DateTime).TotalMinutes : 0);
+      return sum;
+    }
+  }
+
+  public static class LinkedListNodeExtensions {
+    public static IEnumerable<LinkedListNode<T>> EnumerateNodes<T>(this LinkedList<T> list) {
+      var node = list.First;
+      while (node != null) {
+        yield return node;
+        node = node.Next;
+      }
+    }
+
+    public static bool NextIs<T>(this LinkedListNode<T> node, Predicate<T> predicate) {
+      return node.Next != null && predicate(node.Next.Value);
     }
   }