Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs @ 9541

Last change on this file since 9541 was 9541, checked in by pfleck, 11 years ago

#2063:
UserId in ClientInfo fact table is nullable. UserId is removed from primary key therefore.
StatisticsGenerator insert new ClientInfo into fact table.

File size: 7.3 KB
Line 
1using System;
2using System.Linq;
3using System.Transactions;
4using HeuristicLab.Services.Hive.DataAccess;
5
6namespace HeuristicLab.Services.Hive {
7  public class HiveStatisticsGenerator : IStatisticsGenerator {
8    private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
9
10    public void GenerateStatistics() {
11      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
12      using (var transaction = new TransactionScope()) {
13        var newTime = UpdateDimensionTables(db);
14        db.SubmitChanges();
15
16        if (newTime != null) {
17          UpdateFactTables(newTime, db);
18          db.SubmitChanges();
19        }
20
21        transaction.Complete();
22      }
23    }
24
25    private DimTime UpdateDimensionTables(HiveDataContext db) {
26      var newTime = UpdateTime(db);
27      // Update other tables out of sync with time dimension?
28      UpdateJobs(db);
29      UpdateUsers(db);
30      UpdateClients(db);
31
32      return newTime;
33    }
34
35    private DimTime UpdateTime(HiveDataContext db) {
36      var lastUpdate =
37        (from t in db.DimTimes
38         orderby t.Time descending
39         select t.Time)
40        .FirstOrDefault();
41
42      var now = DateTime.Now;
43      DimTime newTime = null;
44      if (lastUpdate == default(DateTime) || lastUpdate + SmallestTimeSpan < now) {
45        newTime = new DimTime {
46          Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute - now.Minute % SmallestTimeSpan.Minutes, 0),
47          Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
48          Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
49          Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
50          Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
51        };
52        db.DimTimes.InsertOnSubmit(newTime);
53      }
54
55      return newTime;
56    }
57
58    private void UpdateJobs(HiveDataContext db) {
59      var newJobs =
60        from j in db.Jobs
61        where !db.DimJobs.Select(x => x.JobId).Contains(j.JobId)
62        select j;
63
64      var newDimJobs =
65        from j in newJobs.ToList()
66        select new DimJob {
67          JobId = j.JobId,
68          JobName = j.Name,
69          UserId = j.OwnerUserId,
70          UserName = ""
71        };
72
73      db.DimJobs.InsertAllOnSubmit(newDimJobs);
74    }
75
76    private void UpdateUsers(HiveDataContext db) {
77      var newUsers =
78        from u in db.Resources.Where(x => x.OwnerUserId != null).Select(x => x.OwnerUserId.Value).Union(db.Jobs.Select(x => x.OwnerUserId))
79        where !db.DimUsers.Select(x => x.UserId).Contains(u)
80        select u;
81
82      var newDimUsers =
83        from u in newUsers.ToList()
84        select new DimUser {
85          UserId = u,
86          Name = ""
87        };
88
89      db.DimUsers.InsertAllOnSubmit(newDimUsers);
90    }
91
92    private void UpdateClients(HiveDataContext db) {
93      var removedClients =
94        from c in db.DimClients
95        where c.ExpirationTime == null &&
96              !db.Resources.OfType<Slave>().Select(x => x.ResourceId).Contains(c.ResourceId)
97        select c;
98
99      var modifiedClients =
100        from s in db.Resources.OfType<Slave>()
101        join c in db.DimClients on s.ResourceId equals c.ResourceId
102        where c.ExpirationTime == null
103              && (s.Name != c.Name || s.ParentResourceId != c.ResourceGroupId ||
104                  s.ParentResource.ParentResourceId != c.ResourceGroup2Id)
105        select new { Slave = s, Client = c };
106
107      foreach (var client in removedClients.Union(modifiedClients.Select(x => x.Client))) {
108        client.ExpirationTime = DateTime.Now;
109      }
110
111      var newClients =
112        from s in db.Resources.OfType<Slave>()
113        where !db.DimClients.Select(x => x.ResourceId).Contains(s.ResourceId)
114          || modifiedClients.Select(x => x.Slave.ResourceId).Contains(s.ResourceId)
115        select new {
116          Slave = s,
117          Group = s.ParentResourceId,
118          Group2 = s.ParentResource.ParentResourceId
119        };
120
121      var newDimClients =
122        from s in newClients.ToList()
123        select new DimClient {
124          ResourceId = s.Slave.ResourceId,
125          Name = s.Slave.Name,
126          ExpirationTime = null,
127          ResourceGroupId = s.Group,
128          ResourceGroup2Id = s.Group2
129        };
130
131      db.DimClients.InsertAllOnSubmit(newDimClients);
132    }
133
134    private void UpdateFactTables(DimTime newTime, HiveDataContext db) {
135      UpdateClientInfoFacts(newTime, db);
136      //UpdateTaskFacts(newTime, db);
137    }
138
139    private void UpdateClientInfoFacts(DimTime newTime, HiveDataContext db) {
140      var lastFacts =
141        from cf in db.FactClientInfos
142        group cf by cf.ClientId
143          into grpFacts
144          select grpFacts.OrderByDescending(x => x.Time).First();
145
146      var slaves =
147        from s in db.Resources.OfType<Slave>()
148        join c in db.DimClients on s.ResourceId equals c.ResourceId
149        join lcf in lastFacts on c.Id equals lcf.ClientId into joinCf
150        from cf in joinCf.DefaultIfEmpty()
151        select new {
152          Slave = s,
153          Client = c,
154          LastFact = cf
155        };
156
157      var clientFacts =
158        from s in slaves.ToList()
159        select new FactClientInfo {
160          DimClient = s.Client,
161          DimTime = newTime,
162          UserId = s.Slave.OwnerUserId,
163          NumUsedCores =
164            s.Slave.Cores != null && s.Slave.FreeCores != null
165              ? s.Slave.Cores.Value - s.Slave.FreeCores.Value
166              : 0,
167          NumTotalCores = s.Slave.Cores ?? 0,
168          UsedMemory =
169            s.Slave.Memory != null && s.Slave.FreeMemory != null
170              ? s.Slave.Memory.Value - s.Slave.FreeMemory.Value
171              : 0,
172          TotalMemory = s.Slave.Memory ?? 0,
173          CpuUtilization = s.Slave.CpuUtilization,
174          TrafficIn = 0,
175          TrafficOut = 0,
176          TotalTimeIdle = CalcNewTotalTime(s.LastFact, newTime.Time,
177                                           x => x.TotalTimeIdle,
178                                           () => s.Slave.SlaveState == SlaveState.Idle && s.Slave.IsAllowedToCalculate),
179          TotalTimeCalculating = CalcNewTotalTime(s.LastFact, newTime.Time,
180                                                  x => x.TotalTimeCalculating,
181                                                  () => s.Slave.SlaveState == SlaveState.Calculating),
182          TotalTimeTransferring = 0.0,
183          TotalTimeUnavailable = CalcNewTotalTime(s.LastFact, newTime.Time,
184                                                  x => x.TotalTimeUnavailable,
185                                                  () => s.Slave.SlaveState == SlaveState.Idle && !s.Slave.IsAllowedToCalculate),
186          TotalTimeOffline = CalcNewTotalTime(s.LastFact, newTime.Time,
187                                              x => x.TotalTimeOffline,
188                                              () => s.Slave.SlaveState == SlaveState.Offline)
189        };
190
191      db.FactClientInfos.InsertAllOnSubmit(clientFacts);
192    }
193
194    private double CalcNewTotalTime(FactClientInfo lastFact, DateTime newTime, Func<FactClientInfo, double> selector, Func<bool> condition) {
195      if (lastFact == null) {
196        return 0.0;
197      }
198      return condition()
199               ? selector(lastFact) + (newTime - lastFact.Time).TotalMinutes
200               : selector(lastFact);
201    }
202  }
203}
Note: See TracBrowser for help on using the repository browser.