Changeset 9541 for branches/HiveStatistics/sources

Timestamp:

05/27/13 17:03:43 (12 years ago)

Author:

pfleck

Message:

#2063:
UserId in ClientInfo fact table is nullable. UserId is removed from primary key therefore.
StatisticsGenerator insert new ClientInfo into fact table.

Location:

branches/HiveStatistics/sources

Files:

• HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml (modified) (1 diff)
• HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml.layout (modified) (7 diffs)
• HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs (modified) (4 diffs)
• HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql (modified) (2 diffs)
• HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs (modified) (3 diffs)

branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml

@@ -277 +277 @@
       <Column Name="ClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" CanBeNull="false" />
       <Column Name="Time" Type="System.DateTime" DbType="DateTime NOT NULL" IsPrimaryKey="true" CanBeNull="false" />
-      <Column Name="UserId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" CanBeNull="false" />
+      <Column Name="UserId" Type="System.Guid" DbType="UniqueIdentifier NULL" CanBeNull="true" />
       <Column Name="NumUsedCores" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
       <Column Name="NumTotalCores" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />

branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml.layout

@@ -121 +121 @@
       </nestedChildShapes>
     </classShape>
-    <associationConnector edgePoints="[(11.25 : 1.87677124023437); (10.9375 : 1.87677124023437); (10.9375 : 0.6875); (2.75 : 0.6875); (2.75 : 1)]" fixedFrom="NotFixed" fixedTo="NotFixed">
+    <associationConnector edgePoints="[(11.25 : 1.44547136382195); (11.0260416666667 : 1.44547136382195); (11.0260416666667 : 0.5625); (2.75 : 0.5625); (2.75 : 1)]" manuallyRouted="true" fixedFrom="Caller" fixedTo="Algorithm">
       <AssociationMoniker Name="/HiveDataContext/Resource/Resource_StateLog" />
       <nodes>
@@ -179 +179 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(7.96163946463664 : 4.30930826822917); (7.96163946463664 : 5.09699625651042); (8.875 : 5.09699625651042)]" fixedFrom="NotFixed" fixedTo="NotFixed">
+    <associationConnector edgePoints="[(8.0192961430407 : 3.90397379557292); (8.0192961430407 : 5.04807657877604); (8.875 : 5.04807657877604)]" fixedFrom="Caller" fixedTo="Algorithm">
       <AssociationMoniker Name="/HiveDataContext/Task/Task_AssignedResource" />
       <nodes>
@@ -186 +186 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(7.16831973231832 : 4.30930826822917); (7.16831973231832 : 5.875)]" fixedFrom="NotFixed" fixedTo="NotFixed">
+    <associationConnector edgePoints="[(7.20145120722822 : 3.90397379557292); (7.20145120722822 : 5.875)]" fixedFrom="Caller" fixedTo="Algorithm">
       <AssociationMoniker Name="/HiveDataContext/Task/Task_RequiredPlugin" />
       <nodes>
@@ -193 +193 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(7.20923076923077 : 1); (7.20923076923077 : 0.875); (7.80461538461539 : 0.875); (7.80461538461539 : 1)]" fixedFrom="NotFixed" fixedTo="NotFixed">
+    <associationConnector edgePoints="[(7.20923076923077 : 1); (7.20923076923077 : 0.770833333333333); (7.80461538461539 : 0.770833333333333); (7.80461538461539 : 1)]" manuallyRouted="true" fixedFrom="Algorithm" fixedTo="Algorithm">
       <AssociationMoniker Name="/HiveDataContext/Task/Task_Task" />
       <nodes>
@@ -297 +297 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(5.15625 : 10.0891951497396); (5.15625 : 14.3889184570313); (6.875 : 14.3889184570313)]" fixedFrom="Algorithm" fixedTo="Algorithm">
+    <associationConnector edgePoints="[(5.15625 : 10.0891951497396); (5.15625 : 14.3889184570313); (6.875 : 14.3889184570313)]" fixedFrom="NotFixed" fixedTo="NotFixed">
       <AssociationMoniker Name="/HiveDataContext/DimTime/DimTime_FactTask" />
       <nodes>
@@ -304 +304 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(5.4218775 : 10.0891951497396); (5.4218775 : 13.3576659570313); (6.71675495975 : 13.3576659570313); (6.71675495975 : 13.359301744963); (6.875 : 13.359301744963)]" manuallyRouted="true" fixedFrom="Algorithm" fixedTo="Caller">
+    <associationConnector edgePoints="[(5.4218775 : 10.0891951497396); (5.4218775 : 13.3576659570313); (6.71675495975 : 13.3576659570313); (6.71675495975 : 13.359301744963); (6.875 : 13.359301744963)]" manuallyRouted="true" fixedFrom="NotFixed" fixedTo="NotFixed">
       <AssociationMoniker Name="/HiveDataContext/DimTime/DimTime_FactTask1" />
       <nodes>
@@ -318 +318 @@
       </nodes>
     </associationConnector>
-    <associationConnector edgePoints="[(4.125 : 10.0891951497396); (4.125 : 11)]" fixedFrom="Algorithm" fixedTo="Algorithm">
+    <associationConnector edgePoints="[(4.125 : 10.0891951497396); (4.125 : 11)]" fixedFrom="NotFixed" fixedTo="NotFixed">
       <AssociationMoniker Name="/HiveDataContext/DimTime/DimTime_FactClientInfo" />
       <nodes>

branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs

@@ -5929 +5929 @@
     private System.DateTime _Time;
     
-    private System.Guid _UserId;
+    private System.Nullable<System.Guid> _UserId;
     
     private int _NumUsedCores;
@@ -5969 +5969 @@
     partial void OnTimeChanging(System.DateTime value);
     partial void OnTimeChanged();
-    partial void OnUserIdChanging(System.Guid value);
+    partial void OnUserIdChanging(System.Nullable<System.Guid> value);
     partial void OnUserIdChanged();
     partial void OnNumUsedCoresChanging(int value);
@@ -6053 +6053 @@
     }
     
-    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_UserId", DbType="UniqueIdentifier NOT NULL", IsPrimaryKey=true)]
-    public System.Guid UserId
+    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_UserId", DbType="UniqueIdentifier NULL")]
+    public System.Nullable<System.Guid> UserId
     {
       get
@@ -6412 +6412 @@
           else
           {
-            this._UserId = default(System.Guid);
+            this._UserId = default(Nullable<System.Guid>);
           }
           this.SendPropertyChanged("DimUser");

branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql

@@ -233 +233 @@
     [ClientId]              UNIQUEIDENTIFIER NOT NULL,
     [Time]                  DATETIME         NOT NULL,
-    [UserId]                UNIQUEIDENTIFIER NOT NULL,
+    [UserId]                UNIQUEIDENTIFIER NULL,
     [NumUsedCores]          INT              NOT NULL,
     [NumTotalCores]         INT              NOT NULL,
@@ -246 +246 @@
     [TotalTimeOffline]      FLOAT (53)       NOT NULL,
     [TotalTimeUnavailable]  FLOAT (53)       NOT NULL,
-    CONSTRAINT [PK_FactClientInfo] PRIMARY KEY CLUSTERED ([ClientId] ASC, [Time] ASC, [UserId] ASC),
+    CONSTRAINT [PK_FactClientInfo] PRIMARY KEY CLUSTERED ([ClientId] ASC, [Time] ASC),
     CONSTRAINT [FK_FactClientInfo_DimTime] FOREIGN KEY ([Time]) REFERENCES [statistics].[DimTime] ([Time]),
     CONSTRAINT [FK_FactClientInfo_DimClient] FOREIGN KEY ([ClientId]) REFERENCES [statistics].[DimClient] ([Id]),

branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

@@ -6 +6 @@
 namespace HeuristicLab.Services.Hive {
   public class HiveStatisticsGenerator : IStatisticsGenerator {
-    private static readonly TimeSpan SmallesTimeSpan = new TimeSpan(0, 5, 0);
+    private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
 
     public void GenerateStatistics() {
       using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
       using (var transaction = new TransactionScope()) {
-
         var newTime = UpdateDimensionTables(db);
+        db.SubmitChanges();
+
         if (newTime != null) {
-          UpdateDataTables(newTime, db);
+          UpdateFactTables(newTime, db);
+          db.SubmitChanges();
         }
 
-        db.SubmitChanges();
         transaction.Complete();
       }
@@ -41 +42 @@
       var now = DateTime.Now;
       DimTime newTime = null;
-      if (lastUpdate == default(DateTime) || lastUpdate + SmallesTimeSpan < now) {
+      if (lastUpdate == default(DateTime) || lastUpdate + SmallestTimeSpan < now) {
         newTime = new DimTime {
-          Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute - now.Minute % SmallesTimeSpan.Minutes, 0),
+          Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute - now.Minute % SmallestTimeSpan.Minutes, 0),
           Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
           Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
@@ -131 +132 @@
     }
 
-    private void UpdateDataTables(DimTime newTime, HiveDataContext db) {
-
+    private void UpdateFactTables(DimTime newTime, HiveDataContext db) {
+      UpdateClientInfoFacts(newTime, db);
+      //UpdateTaskFacts(newTime, db);
+    }
+
+    private void UpdateClientInfoFacts(DimTime newTime, HiveDataContext db) {
+      var lastFacts =
+        from cf in db.FactClientInfos
+        group cf by cf.ClientId
+          into grpFacts
+          select grpFacts.OrderByDescending(x => x.Time).First();
+
+      var slaves =
+        from s in db.Resources.OfType<Slave>()
+        join c in db.DimClients on s.ResourceId equals c.ResourceId
+        join lcf in lastFacts on c.Id equals lcf.ClientId into joinCf
+        from cf in joinCf.DefaultIfEmpty()
+        select new {
+          Slave = s,
+          Client = c,
+          LastFact = cf
+        };
+
+      var clientFacts =
+        from s in slaves.ToList()
+        select new FactClientInfo {
+          DimClient = s.Client,
+          DimTime = newTime,
+          UserId = s.Slave.OwnerUserId,
+          NumUsedCores =
+            s.Slave.Cores != null && s.Slave.FreeCores != null
+              ? s.Slave.Cores.Value - s.Slave.FreeCores.Value
+              : 0,
+          NumTotalCores = s.Slave.Cores ?? 0,
+          UsedMemory =
+            s.Slave.Memory != null && s.Slave.FreeMemory != null
+              ? s.Slave.Memory.Value - s.Slave.FreeMemory.Value
+              : 0,
+          TotalMemory = s.Slave.Memory ?? 0,
+          CpuUtilization = s.Slave.CpuUtilization,
+          TrafficIn = 0,
+          TrafficOut = 0,
+          TotalTimeIdle = CalcNewTotalTime(s.LastFact, newTime.Time,
+                                           x => x.TotalTimeIdle,
+                                           () => s.Slave.SlaveState == SlaveState.Idle && s.Slave.IsAllowedToCalculate),
+          TotalTimeCalculating = CalcNewTotalTime(s.LastFact, newTime.Time,
+                                                  x => x.TotalTimeCalculating,
+                                                  () => s.Slave.SlaveState == SlaveState.Calculating),
+          TotalTimeTransferring = 0.0,
+          TotalTimeUnavailable = CalcNewTotalTime(s.LastFact, newTime.Time,
+                                                  x => x.TotalTimeUnavailable,
+                                                  () => s.Slave.SlaveState == SlaveState.Idle && !s.Slave.IsAllowedToCalculate),
+          TotalTimeOffline = CalcNewTotalTime(s.LastFact, newTime.Time,
+                                              x => x.TotalTimeOffline,
+                                              () => s.Slave.SlaveState == SlaveState.Offline)
+        };
+
+      db.FactClientInfos.InsertAllOnSubmit(clientFacts);
+    }
+
+    private double CalcNewTotalTime(FactClientInfo lastFact, DateTime newTime, Func<FactClientInfo, double> selector, Func<bool> condition) {
+      if (lastFact == null) {
+        return 0.0;
+      }
+      return condition()
+               ? selector(lastFact) + (newTime - lastFact.Time).TotalMinutes
+               : selector(lastFact);
     }
   }