Changeset 12468 for branches/HiveStatistics
- Timestamp:
- 06/18/15 11:58:58 (10 years ago)
- Location:
- branches/HiveStatistics/sources
- Files:
-
- 31 added
- 6 deleted
- 14 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HeuristicLab.Services.Hive.DataAccess-3.3.csproj
r11623 r12468 104 104 <ItemGroup> 105 105 <None Include="Plugin.cs.frame" /> 106 <Compile Include="Daos\AssignedResourceDao.cs" /> 107 <Compile Include="Daos\DowntimeDao.cs" /> 108 <Compile Include="Daos\GenericDao.cs" /> 109 <Compile Include="Daos\HiveStatistics\DimClientDao.cs" /> 110 <Compile Include="Daos\HiveStatistics\DimJobDao.cs" /> 111 <Compile Include="Daos\HiveStatistics\DimTimeDao.cs" /> 112 <Compile Include="Daos\HiveStatistics\DimUserDao.cs" /> 113 <Compile Include="Daos\HiveStatistics\FactClientInfoDao.cs" /> 114 <Compile Include="Daos\HiveStatistics\FactTaskDao.cs" /> 115 <Compile Include="Daos\JobDao.cs" /> 116 <Compile Include="Daos\JobPermissionDao.cs" /> 117 <Compile Include="Daos\LifecycleDao.cs" /> 118 <Compile Include="Daos\PluginDao.cs" /> 119 <Compile Include="Daos\PluginDataDao.cs" /> 120 <Compile Include="Daos\RequiredPluginDao.cs" /> 121 <Compile Include="Daos\ResourceDao.cs" /> 122 <Compile Include="Daos\ResourcePermissionDao.cs" /> 123 <Compile Include="Daos\SlaveDao.cs" /> 124 <Compile Include="Daos\SlaveGroupDao.cs" /> 125 <Compile Include="Daos\StateLogDao.cs" /> 126 <Compile Include="Daos\TaskDao.cs" /> 127 <Compile Include="Daos\TaskDataDao.cs" /> 128 <Compile Include="Daos\UserPriorityDao.cs" /> 129 <Compile Include="Interfaces\IGenericDao.cs" /> 106 130 <Compile Include="Enums\Command.cs" /> 107 131 <Compile Include="Enums\DowntimeType.cs" /> 108 132 <Compile Include="Enums\CpuArchitecture.cs" /> 133 <Compile Include="Interfaces\IPersistenceManager.cs" /> 134 <Compile Include="Manager\PersistenceManager.cs" /> 109 135 <Compile Include="Plugin.cs" /> 110 136 <Compile Include="HiveDataContext.cs"> … … 116 142 <DependentUpon>HiveDataContext.dbml</DependentUpon> 117 143 </Compile> 118 <Compile Include="I TransactionManager.cs" />144 <Compile Include="Interfaces\ITransactionManager.cs" /> 119 145 <Compile Include="Enums\TaskState.cs" /> 120 146 <Compile Include="Logger.cs" /> … … 128 154 </Compile> 129 155 <Compile Include="Enums\SlaveState.cs" /> 130 <Compile Include=" TransactionManager.cs" />156 <Compile Include="Manager\TransactionManager.cs" /> 131 157 </ItemGroup> 132 158 <ItemGroup> … … 158 184 <Content Include="SQL Scripts\Prepare Hive Database.sql" /> 159 185 </ItemGroup> 186 <ItemGroup /> 160 187 <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> 161 188 <PropertyGroup> -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml
r12441 r12468 1 1 <?xml version="1.0" encoding="utf-8"?><Database Name="HeuristicLab.Hive-3.3" Class="HiveDataContext" xmlns="http://schemas.microsoft.com/linqtosql/dbml/2007"> 2 <Connection Mode="AppSettings" ConnectionString="Data Source= .\SQLEXPRESS;Initial Catalog=HeuristicLab.Hive-3.3;Integrated Security=True" SettingsObjectName="HeuristicLab.Services.Hive.DataAccess.Settings" SettingsPropertyName="HeuristicLab_Hive_LinqConnectionString" Provider="System.Data.SqlClient" />2 <Connection Mode="AppSettings" ConnectionString="Data Source=localhost;Initial Catalog=HeuristicLab.Hive-3.3;Integrated Security=True" SettingsObjectName="HeuristicLab.Services.Hive.DataAccess.Settings" SettingsPropertyName="HeuristicLab_Hive_LinqConnectionString" Provider="System.Data.SqlClient" /> 3 3 <Table Name="dbo.AssignedResources" Member="AssignedResources"> 4 4 <Type Name="AssignedResource"> … … 233 233 <Column Name="MemoryRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" /> 234 234 <Column Name="Priority" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" /> 235 <Column Name="LastClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull=" false" />235 <Column Name="LastClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="true" /> 236 236 <Column Name="JobId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="false" /> 237 237 <Column Name="StartTime" Type="System.DateTime" DbType="DateTime NOT NULL" CanBeNull="false" /> -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml.layout
r12441 r12468 297 297 </nodes> 298 298 </associationConnector> 299 <associationConnector edgePoints="[(5.02343625 : 10.2708968098958); (5.02343625 : 14.7227579166667); ( 6.875 : 14.7227579166667)]" fixedFrom="NotFixed" fixedTo="NotFixed">299 <associationConnector edgePoints="[(5.02343625 : 10.2708968098958); (5.02343625 : 14.7227579166667); (5.90885291666667 : 14.7227579166667 : JumpStart); (6.07551958333333 : 14.7227579166667 : JumpEnd); (6.875 : 14.7227579166667)]" fixedFrom="NotFixed" fixedTo="NotFixed"> 300 300 <AssociationMoniker Name="/HiveDataContext/DimTime/DimTime_FactTask" /> 301 301 <nodes> -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs
r12441 r12468 3 3 // <auto-generated> 4 4 // This code was generated by a tool. 5 // Runtime Version:4.0.30319.34 0145 // Runtime Version:4.0.30319.34209 6 6 // 7 7 // Changes to this file may cause incorrect behavior and will be lost if … … 4863 4863 private int _Priority; 4864 4864 4865 private System. Guid_LastClientId;4865 private System.Nullable<System.Guid> _LastClientId; 4866 4866 4867 4867 private System.Guid _JobId; … … 4905 4905 partial void OnPriorityChanging(int value); 4906 4906 partial void OnPriorityChanged(); 4907 partial void OnLastClientIdChanging(System. Guidvalue);4907 partial void OnLastClientIdChanging(System.Nullable<System.Guid> value); 4908 4908 partial void OnLastClientIdChanged(); 4909 4909 partial void OnJobIdChanging(System.Guid value); … … 5109 5109 5110 5110 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_LastClientId", DbType="UniqueIdentifier NOT NULL")] 5111 public System. GuidLastClientId5111 public System.Nullable<System.Guid> LastClientId 5112 5112 { 5113 5113 get … … 5271 5271 else 5272 5272 { 5273 this._LastClientId = default( System.Guid);5273 this._LastClientId = default(Nullable<System.Guid>); 5274 5274 } 5275 5275 this.SendPropertyChanged("DimClient"); -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql
r12441 r12468 239 239 [TotalMemory] INT NOT NULL, 240 240 [CpuUtilization] FLOAT (53) NOT NULL, 241 241 [SlaveState] VarChar(15) NOT NULL, 242 242 [TotalTimeIdle] FLOAT (53) NOT NULL, 243 243 [TotalTimeCalculating] FLOAT (53) NOT NULL, … … 260 260 [MemoryRequired] INT NOT NULL, 261 261 [Priority] INT NOT NULL, 262 [LastClientId] UNIQUEIDENTIFIER N OT NULL,262 [LastClientId] UNIQUEIDENTIFIER NULL, 263 263 [JobId] UNIQUEIDENTIFIER NOT NULL, 264 264 [StartTime] DATETIME NOT NULL, 265 265 [EndTime] DATETIME NULL, 266 267 266 [TaskState] VARCHAR (30) NOT NULL, 267 [Exception] VARCHAR (MAX) NULL, 268 268 CONSTRAINT [PK_FactTask] PRIMARY KEY CLUSTERED ([TaskId] ASC), 269 269 CONSTRAINT [FK_FactTask_DimClient] FOREIGN KEY ([LastClientId]) REFERENCES [statistics].[DimClient] ([Id]), … … 273 273 ); 274 274 275 INSERT INTO [HeuristicLab.Hive-3.3].[statistics].[DimClient] ([Id] ,[Name] ,[ResourceId]) 276 VALUES ('00000000-0000-0000-0000-000000000000' ,'NULL' ,'00000000-0000-0000-0000-000000000000') 275 /* dummy for nullable userIds in FactClientInfo */ 276 INSERT INTO [statistics].[DimUser] ([UserId], [Name]) 277 VALUES ('00000000-0000-0000-0000-000000000000', 'NULL'); -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.Web/Hive-3.3/Status.aspx.cs
r12146 r12468 84 84 85 85 overallCpuUtilizationLabel.Text = (onlineSlaves.Any() ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2).ToString() : "0.0") + " %"; 86 cpuUtilizationLabel.Text = (onlineSlaves.Any( ) && onlineSlaves.Any(x => x.IsAllowedToCalculate) ? Math.Round(onlineSlaves.Where(x => x.IsAllowedToCalculate).Average(s => s.CpuUtilization), 2).ToString() : "0.0") + " %";86 cpuUtilizationLabel.Text = (onlineSlaves.Any(x => x.IsAllowedToCalculate) ? Math.Round(onlineSlaves.Where(x => x.IsAllowedToCalculate).Average(s => s.CpuUtilization), 2).ToString() : "0.0") + " %"; 87 87 88 88 DT.Statistics[] stats = new DT.Statistics[0]; -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveDao.cs
r12146 r12468 891 891 public List<DT.UserStatistics> GetUserStatistics() { 892 892 using (var db = CreateContext()) { 893 db.Log = Console.Out; 893 894 var userStats = new Dictionary<Guid, DT.UserStatistics>(); 894 895 -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveJanitor.cs
r12441 r12468 23 23 using System.Threading; 24 24 using HeuristicLab.Services.Hive.DataAccess; 25 using HeuristicLab.Services.Hive.DataAccess.Interfaces; 25 26 26 27 namespace HeuristicLab.Services.Hive { … … 30 31 private AutoResetEvent generateStatisticsWaitHandle; 31 32 32 private DataAccess.ITransactionManager trans {33 private ITransactionManager trans { 33 34 get { return ServiceLocator.Instance.TransactionManager; } 34 35 } … … 55 56 stop = true; 56 57 cleanupWaitHandle.Set(); 58 generateStatisticsWaitHandle.Set(); 57 59 } 58 60 -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveService.cs
r12012 r12468 24 24 using System.Linq; 25 25 using System.ServiceModel; 26 using HeuristicLab.Services.Hive.DataAccess.Interfaces; 26 27 using HeuristicLab.Services.Hive.DataTransfer; 27 28 using HeuristicLab.Services.Hive.ServiceContracts; … … 51 52 get { return ServiceLocator.Instance.AuthorizationManager; } 52 53 } 53 private DataAccess.ITransactionManager trans {54 private ITransactionManager trans { 54 55 get { return ServiceLocator.Instance.TransactionManager; } 55 56 } -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs
r12441 r12468 1 using System; 1 #region License Information 2 /* HeuristicLab 3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL) 4 * 5 * This file is part of HeuristicLab. 6 * 7 * HeuristicLab is free software: you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License as published by 9 * the Free Software Foundation, either version 3 of the License, or 10 * (at your option) any later version. 11 * 12 * HeuristicLab is distributed in the hope that it will be useful, 13 * but WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 * GNU General Public License for more details. 16 * 17 * You should have received a copy of the GNU General Public License 18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>. 19 */ 20 #endregion 21 22 using System; 2 23 using System.Collections.Generic; 3 24 using System.Data.Linq; 4 25 using System.Linq; 5 using System.Transactions;6 using HeuristicLab.Services.Access;7 26 using HeuristicLab.Services.Hive.DataAccess; 27 using HeuristicLab.Services.Hive.DataAccess.Manager; 8 28 9 29 namespace HeuristicLab.Services.Hive { 10 30 public class HiveStatisticsGenerator : IStatisticsGenerator { 11 private IUserManager userManager { get { return ServiceLocator.Instance.UserManager; } } 12 31 32 private const string UnknownUserName = "Unknown"; 13 33 private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0); 14 private static readonly string UnknownUserName = "Unknown";15 34 16 35 public void GenerateStatistics() { 17 using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString)) 18 using (var transaction = new TransactionScope(TransactionScopeOption.Required, new TransactionOptions { IsolationLevel = IsolationLevel.ReadCommitted })) { 19 20 var newTime = UpdateDimensionTables(db); 21 db.SubmitChanges(); 22 23 if (newTime != null) { 24 UpdateFactTables(newTime, db); 25 try { 26 db.SubmitChanges(); 36 using (var pm = new PersistenceManager()) { 37 pm.UseTransaction(() => { 38 UpdateDimUserTable(pm); 39 UpdateDimJobTable(pm); 40 UpdateDimClients(pm); 41 pm.SubmitChanges(); 42 }); 43 44 pm.UseTransaction(() => { 45 var newTime = UpdateDimTimeTable(pm); 46 if (newTime != null) { 47 pm.SubmitChanges(); 48 UpdateFactClientInfoTable(newTime, pm); 49 UpdateTaskFacts(newTime, pm); 50 try { 51 pm.SubmitChanges(); 52 } 53 catch (DuplicateKeyException e) { 54 var logger = LogFactory.GetLogger(typeof(HiveStatisticsGenerator).Namespace); 55 logger.Log(string.Format( 56 @"Propable change from summertime to wintertime, resulting in overlapping times. 57 On wintertime to summertime change, slave timeouts and a fact gap will occur. 58 Exception Details: {0}", e)); 59 } 27 60 } 28 catch (DuplicateKeyException) { 29 // Propable change from summertime to wintertime, resulting in overlapping times 30 // On wintertime to summertime change, slave timeouts and a fact gap will occur 31 } 61 }); 62 } 63 } 64 65 private DimTime UpdateDimTimeTable(PersistenceManager pm) { 66 var dimTimeDao = pm.DimTimeDao; 67 var now = DateTime.Now; 68 var timeEntry = new DimTime { 69 Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0), 70 Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0), 71 Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0), 72 Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0), 73 Year = new DateTime(now.Year, 1, 1, 0, 0, 0) 74 }; 75 // using SaveOrAttach instead of save because the table can only be updated once every minute 76 // (pk restriction) save would cause a duplicate key exception if the method is accidentally 77 // called more than once every minute (e.g. restarting the service within a minute) 78 return dimTimeDao.SaveOrAttach(timeEntry); 79 } 80 81 private void UpdateDimUserTable(PersistenceManager pm) { 82 var dimUserDao = pm.DimUserDao; 83 var resourceDao = pm.ResourceDao; 84 var jobDao = pm.JobDao; 85 var existingUserIds = dimUserDao.GetAll().Select(x => x.UserId); 86 var vaildResourceOwnerIds = resourceDao.GetResourcesWithValidOwner().Select(x => x.OwnerUserId.Value); 87 var jobOwnerIds = jobDao.GetAll().Select(x => x.OwnerUserId); 88 var newUserIds = vaildResourceOwnerIds 89 .Union(jobOwnerIds) 90 .Where(id => !existingUserIds.Contains(id)) 91 .ToList(); 92 dimUserDao.Save(newUserIds.Select(x => new DimUser { 93 UserId = x, 94 Name = GetUserName(x) 95 })); 96 } 97 98 private void UpdateDimJobTable(PersistenceManager pm) { 99 var dimJobDao = pm.DimJobDao; 100 var jobDao = pm.JobDao; 101 var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId); 102 var newJobs = jobDao.GetAll() 103 .Where(x => !dimJobIds.Contains(x.JobId)) 104 .Select(x => new { 105 JobId = x.JobId, 106 UserId = x.OwnerUserId, 107 JobName = x.Name ?? string.Empty, 108 }) 109 .ToList(); 110 dimJobDao.Save(newJobs.Select(x => new DimJob { 111 JobId = x.JobId, 112 JobName = x.JobName, 113 UserId = x.UserId, 114 UserName = GetUserName(x.UserId) 115 })); 116 } 117 118 private void UpdateDimClients(PersistenceManager pm) { 119 var dimClientDao = pm.DimClientDao; 120 var slaveDao = pm.SlaveDao; 121 var slaves = slaveDao.GetAll(); 122 var recentlyAddedClients = dimClientDao.GetRecentlyAddedClients(); 123 var slaveIds = slaves.Select(x => x.ResourceId); 124 125 var removedClientIds = recentlyAddedClients 126 .Where(x => !slaveIds.Contains(x.ResourceId)) 127 .Select(x => x.Id); 128 var modifiedClients = 129 from slave in slaves 130 join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId 131 where (slave.Name != client.Name 132 || slave.ParentResourceId != client.ResourceGroupId 133 || slave.ParentResource.ParentResourceId != client.ResourceGroup2Id) 134 select new { 135 SlaveId = slave.ResourceId, 136 ClientId = client.Id 137 }; 138 var clientIds = dimClientDao.GetAll().Select(x => x.ResourceId); 139 var modifiedClientIds = modifiedClients.Select(x => x.SlaveId); 140 var newClients = slaves 141 .Where(x => !clientIds.Contains(x.ResourceId) 142 || modifiedClientIds.Contains(x.ResourceId)) 143 .Select(x => new { 144 x.ResourceId, 145 x.Name, 146 ResourceGroupId = x.ParentResourceId, 147 ResourceGroup2Id = x.ParentResource.ParentResourceId 148 }) 149 .ToList(); 150 151 var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId)); 152 dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now); 153 dimClientDao.Save(newClients.Select(x => new DimClient { 154 ResourceId = x.ResourceId, 155 Name = x.Name, 156 ExpirationTime = null, 157 ResourceGroupId = x.ResourceGroupId, 158 ResourceGroup2Id = x.ResourceGroup2Id 159 })); 160 } 161 162 private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) { 163 var factClientInfoDao = pm.FactClientInfoDao; 164 var slaveDao = pm.SlaveDao; 165 var dimClientDao = pm.DimClientDao; 166 167 var newRawFactInfos = 168 from s in slaveDao.GetAll() 169 join c in dimClientDao.GetRecentlyAddedClients() on s.ResourceId equals c.ResourceId 170 join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf 171 from cf in joinCf.DefaultIfEmpty() 172 select new { 173 ClientId = c.Id, 174 UserId = s.OwnerUserId ?? Guid.Empty, 175 TotalCores = s.Cores ?? 0, 176 FreeCores = s.FreeCores ?? 0, 177 TotalMemory = s.Memory ?? 0, 178 FreeMemory = s.FreeMemory ?? 0, 179 CpuUtilization = s.CpuUtilization, 180 SlaveState = s.SlaveState, 181 IsAllowedToCalculate = s.IsAllowedToCalculate, 182 LastFactTimestamp = cf.Timestamp 183 }; 184 185 factClientInfoDao.Save( 186 from x in newRawFactInfos.ToList() 187 let duration = x.LastFactTimestamp != null 188 ? (newTime.Time - (DateTime)x.LastFactTimestamp).TotalMinutes 189 : SmallestTimeSpan.TotalMinutes 190 select new FactClientInfo { 191 ClientId = x.ClientId, 192 DimTime = newTime, 193 UserId = x.UserId, 194 NumUsedCores = x.TotalCores - x.FreeCores, 195 NumTotalCores = x.TotalMemory, 196 UsedMemory = x.TotalMemory - x.FreeMemory, 197 TotalMemory = x.TotalMemory, 198 CpuUtilization = x.CpuUtilization, 199 SlaveState = x.SlaveState, 200 TotalTimeIdle = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0.0, 201 TotalTimeCalculating = x.SlaveState == SlaveState.Calculating ? duration : 0.0, 202 TotalTimeUnavailable = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0.0, 203 TotalTimeOffline = x.SlaveState == SlaveState.Offline ? duration : 0.0 32 204 } 33 34 transaction.Complete(); 35 } 36 } 37 38 private DimTime UpdateDimensionTables(HiveDataContext db) { 39 var newTime = UpdateTime(db); 40 41 UpdateUsers(db); 42 UpdateJobs(db); 43 UpdateClients(db); 44 45 return newTime; 46 } 47 48 private DimTime UpdateTime(HiveDataContext db) { 49 var lastUpdateTime = 50 (from t in db.DimTimes 51 orderby t.Time descending 52 select t.Time) 53 .FirstOrDefault(); 54 55 var now = DateTime.Now; 56 DimTime newTime = null; 57 58 if (lastUpdateTime == default(DateTime) || lastUpdateTime + SmallestTimeSpan < now) { 59 newTime = new DimTime { 60 Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute - now.Minute % SmallestTimeSpan.Minutes, 0), 61 Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0), 62 Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0), 63 Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0), 64 Year = new DateTime(now.Year, 1, 1, 0, 0, 0) 65 }; 66 db.DimTimes.InsertOnSubmit(newTime); 67 } 68 69 return newTime; 70 } 71 72 private void UpdateUsers(HiveDataContext db) { 73 var newUsers = 74 from u in db.Resources.Where(x => x.OwnerUserId != null).Select(x => x.OwnerUserId.Value).Union(db.Jobs.Select(x => x.OwnerUserId)) 75 where !db.DimUsers.Select(x => x.UserId).Contains(u) 76 select u; 77 78 79 var newDimUsers = 80 from u in newUsers.ToList() 81 select new DimUser { 82 UserId = u, 83 Name = userManager.GetUserById(u) != null ? userManager.GetUserById(u).UserName : UnknownUserName 84 }; 85 86 db.DimUsers.InsertAllOnSubmit(newDimUsers); 87 88 // insert NULL-User 89 if (!db.DimUsers.Any(x => x.UserId == Guid.Empty)) { 90 db.DimUsers.InsertOnSubmit(new DimUser { UserId = Guid.Empty, Name = "NULL" }); 91 } 92 } 93 94 private void UpdateJobs(HiveDataContext db) { 95 var newJobs = 96 from j in db.Jobs 97 where !db.DimJobs.Select(x => x.JobId).Contains(j.JobId) 98 select j; 99 100 var newDimJobs = 101 from j in newJobs.ToList() 102 select new DimJob { 103 JobId = j.JobId, 104 JobName = j.Name == null ? string.Empty : j.Name, 105 UserId = j.OwnerUserId, 106 UserName = userManager.GetUserById(j.OwnerUserId) != null ? userManager.GetUserById(j.OwnerUserId).UserName : UnknownUserName 107 }; 108 109 db.DimJobs.InsertAllOnSubmit(newDimJobs); 110 111 // insert NULL-Job 112 if (!db.DimJobs.Any(x => x.JobId == Guid.Empty)) { 113 db.DimJobs.InsertOnSubmit(new DimJob { JobId = Guid.Empty, JobName = "NULL", UserId = Guid.Empty, UserName = "NULL" }); 114 } 115 } 116 117 private void UpdateClients(HiveDataContext db) { 118 var removedClients = 119 from c in db.DimClients 120 where c.ExpirationTime == null && 121 !db.Resources.OfType<Slave>().Select(x => x.ResourceId).Contains(c.ResourceId) 122 select c; 123 124 var modifiedClients = 125 from s in db.Resources.OfType<Slave>() 126 join c in db.DimClients on s.ResourceId equals c.ResourceId 127 where c.ExpirationTime == null 128 && (s.Name != c.Name || s.ParentResourceId != c.ResourceGroupId || 129 s.ParentResource.ParentResourceId != c.ResourceGroup2Id) 130 select new { Slave = s, Client = c }; 131 132 foreach (var client in removedClients.Union(modifiedClients.Select(x => x.Client))) { 133 client.ExpirationTime = DateTime.Now; 134 } 135 136 var newClients = 137 from s in db.Resources.OfType<Slave>() 138 where !db.DimClients.Select(x => x.ResourceId).Contains(s.ResourceId) 139 || modifiedClients.Select(x => x.Slave.ResourceId).Contains(s.ResourceId) 140 select new { 141 Slave = s, 142 Group = s.ParentResourceId, 143 Group2 = s.ParentResource.ParentResourceId 144 }; 145 146 var newDimClients = 147 from s in newClients.ToList() 148 select new DimClient { 149 ResourceId = s.Slave.ResourceId, 150 Name = s.Slave.Name, 151 ExpirationTime = null, 152 ResourceGroupId = s.Group, 153 ResourceGroup2Id = s.Group2 154 }; 155 156 db.DimClients.InsertAllOnSubmit(newDimClients); 157 } 158 159 private void UpdateFactTables(DimTime newTime, HiveDataContext db) { 160 UpdateClientInfoFacts(newTime, db); 161 UpdateTaskFacts(newTime, db); 162 } 163 164 private void UpdateClientInfoFacts(DimTime newTime, HiveDataContext db) { 165 var time = newTime.Time; 166 167 var lastFacts = 168 from cf in db.FactClientInfos 169 join r in db.DimClients on cf.ClientId equals r.Id 170 group cf by r.ResourceId into grpFacts 171 select new { 172 ResourceId = grpFacts.Key, 173 Fact = grpFacts.OrderByDescending(x => x.Time).First(), 174 }; 175 176 var slaves = 177 from s in db.Resources.OfType<Slave>() 178 join c in db.DimClients on s.ResourceId equals c.ResourceId 179 join lcf in lastFacts on c.ResourceId equals lcf.ResourceId into joinCf 180 from cf in joinCf.DefaultIfEmpty() 181 where c.ExpirationTime == null 182 select new { 183 Slave = s, 184 Client = c, 185 LastFact = cf != null ? cf.Fact : null 186 }; 187 188 var clientFacts = 189 from s in slaves.ToList() 190 let duration = s.LastFact != null ? (time - s.LastFact.Time).TotalMinutes : SmallestTimeSpan.TotalMinutes 191 select new FactClientInfo { 192 DimClient = s.Client, 193 DimTime = newTime, 194 UserId = s.Slave.OwnerUserId ?? Guid.Empty, 195 NumUsedCores = s.Slave.Cores != null && s.Slave.FreeCores != null 196 ? s.Slave.Cores.Value - s.Slave.FreeCores.Value : 0, 197 NumTotalCores = s.Slave.Cores ?? 0, 198 UsedMemory = s.Slave.Memory != null && s.Slave.FreeMemory != null 199 ? s.Slave.Memory.Value - s.Slave.FreeMemory.Value : 0, 200 TotalMemory = s.Slave.Memory ?? 0, 201 CpuUtilization = s.Slave.CpuUtilization, 202 SlaveState = s.Slave.SlaveState, 203 TotalTimeIdle = s.Slave.SlaveState == SlaveState.Idle && s.Slave.IsAllowedToCalculate 204 ? duration : 0.0, 205 TotalTimeCalculating = s.Slave.SlaveState == SlaveState.Calculating 206 ? duration : 0.0, 207 TotalTimeTransferring = 0.0, 208 TotalTimeUnavailable = s.Slave.SlaveState == SlaveState.Idle && !s.Slave.IsAllowedToCalculate 209 ? duration : 0.0, 210 TotalTimeOffline = s.Slave.SlaveState == SlaveState.Offline 211 ? duration : 0.0 212 }; 213 214 db.FactClientInfos.InsertAllOnSubmit(clientFacts); 215 } 216 217 private void UpdateTaskFacts(DimTime newTime, HiveDataContext db) { 218 // old Task facts 219 var oldFacts = 220 from fact in db.FactTasks 221 where fact.EndTime == null 222 select fact; 223 224 // query Task facts data 225 var newFacts = 226 from task in db.Tasks 227 where !task.IsParentTask 228 && (!db.FactTasks.Select(fact => fact.TaskId).Contains(task.TaskId) || oldFacts.Select(fact => fact.TaskId).Contains(task.TaskId)) 229 join lastFact in oldFacts on task.TaskId equals lastFact.TaskId into lastFactPerTask 205 ); 206 } 207 208 private void UpdateTaskFacts(DimTime newTime, PersistenceManager pm) { 209 var factTaskDao = pm.FactTaskDao; 210 var taskDao = pm.TaskDao; 211 var dimClientDao = pm.DimClientDao; 212 var stateLogDao = pm.StateLogDao; 213 214 var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId); 215 var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new { 216 x.TaskId, 217 x.DimTimeStart, 218 x.LastClientId 219 }); 220 221 var newTasks = 222 from task in taskDao.GetAllChildTasks() 223 let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime) 224 let lastSlaveId = stateLogs.First(x => x.SlaveId != null).SlaveId 225 where (!factTaskIds.Contains(task.TaskId) 226 || notFinishedFactTasks.Select(x => x.TaskId).Contains(task.TaskId)) 227 join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask 230 228 from lastFact in lastFactPerTask.DefaultIfEmpty() 231 let lastSlaveId = task.StateLogs.OrderByDescending(sl => sl.DateTime).First(sl => sl.SlaveId != null).SlaveId 232 join client in db.DimClients.Where(client => client.ExpirationTime == null) on lastSlaveId equals client.ResourceId into clientsPerSlaveId 229 join client in dimClientDao.GetRecentlyAddedClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId 233 230 from client in clientsPerSlaveId.DefaultIfEmpty() 234 231 select new { 235 Task = task, 236 StateLogs = task.StateLogs.OrderBy(sl => sl.DateTime), 237 LastClientId = client != null ? client.Id : default(Guid?), 238 LastFact = lastFact 232 TaskId = task.TaskId, 233 JobId = task.JobId, 234 Priority = task.Priority, 235 CoresRequired = task.CoresNeeded, 236 MemoryRequired = task.MemoryNeeded, 237 State = task.State, 238 StateLogs = stateLogs, 239 DimTimeStart = lastFact != null ? lastFact.DimTimeStart : newTime, 240 LastClientId = client != null 241 ? client.Id : lastFact != null 242 ? lastFact.LastClientId : (Guid?)null 239 243 }; 240 244 241 // new Task facts 242 var newTaskFacts = 243 from t in newFacts.ToList() 244 let stateLogsLinkedList = new LinkedList<StateLog>(t.StateLogs) 245 var endStates = new[] { TaskState.Finished, TaskState.Failed, TaskState.Aborted }; 246 factTaskDao.Save( 247 from x in newTasks.ToList() 248 let stateLogsLinkedList = new LinkedList<StateLog>(x.StateLogs.OrderBy(y => y.DateTime)) 249 let lastStateLog = stateLogsLinkedList.OrderByDescending(y => y.DateTime).FirstOrDefault(sl => sl.Exception != null) 245 250 select new FactTask { 246 TaskId = t.Task.TaskId,247 JobId = t.Task.JobId,248 DimTimeStart = t.LastFact != null ? t.LastFact.DimTimeStart : newTime,249 DimTimeEnd = new[] { TaskState.Finished, TaskState.Failed, TaskState.Aborted }.Contains(t.Task.State) ? newTime : null,250 LastClientId = t.LastClientId ?? (t.LastFact != null ? t.LastFact.LastClientId : Guid.Empty),251 Priority = t.Task.Priority,252 CoresRequired = t.Task.CoresNeeded,253 MemoryRequired = t.Task.MemoryNeeded,251 TaskId = x.TaskId, 252 JobId = x.JobId, 253 DimTimeStart = x.DimTimeStart, 254 DimTimeEnd = endStates.Contains(x.State) ? newTime : null, 255 LastClientId = x.LastClientId, 256 Priority = x.Priority, 257 CoresRequired = x.CoresRequired, 258 MemoryRequired = x.MemoryRequired, 254 259 NumCalculationRuns = stateLogsLinkedList.CountCalculationRuns(), 255 260 NumRetries = stateLogsLinkedList.CountRetries(), 256 261 TotalWaitingTime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Waiting), 257 TotalRuntime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs( x => x.State == TaskState.Transferring)),262 TotalRuntime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(y => y.State == TaskState.Transferring)), 258 263 TotalTransferTime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Transferring), 259 TaskState = t.Task.State, 260 Exception = stateLogsLinkedList.FirstOrDefault(sl => sl.Exception != null) == null ? string.Empty : stateLogsLinkedList.FirstOrDefault(sl => sl.Exception != null).Exception 261 }; 262 263 db.FactTasks.DeleteAllOnSubmit(oldFacts.ToList()); 264 db.FactTasks.InsertAllOnSubmit(newTaskFacts); 264 TaskState = x.State, 265 Exception = lastStateLog == null ? string.Empty : lastStateLog.Exception 266 }); 267 factTaskDao.Delete(notFinishedFactTasks.Select(x => x.TaskId)); 268 } 269 270 private string GetUserName(Guid userId) { 271 try { 272 var userManager = ServiceLocator.Instance.UserManager; 273 var user = userManager.GetUserById(userId); 274 return user != null ? user.UserName : UnknownUserName; 275 } 276 catch (Exception) { 277 return UnknownUserName; 278 } 265 279 } 266 280 } -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Interfaces/IServiceLocator.cs
r12441 r12468 21 21 22 22 using HeuristicLab.Services.Hive.DataAccess; 23 using HeuristicLab.Services.Hive.DataAccess.Interfaces; 23 24 24 25 namespace HeuristicLab.Services.Hive { … … 26 27 Access.IRoleVerifier RoleVerifier { get; } 27 28 IAuthorizationManager AuthorizationManager { get; } 29 IPersistenceManager PersistenceManager { get; } 28 30 IHiveDao HiveDao { get; } 29 31 IOptimizedHiveDao OptimizedHiveDao { get; } -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/EventManager.cs
r12172 r12468 24 24 using System.Linq; 25 25 using HeuristicLab.Services.Hive.DataAccess; 26 using HeuristicLab.Services.Hive.DataAccess.Interfaces; 26 27 using DT = HeuristicLab.Services.Hive.DataTransfer; 27 28 … … 41 42 get { return LogFactory.GetLogger(this.GetType().Namespace); } 42 43 } 43 private DataAccess.ITransactionManager trans {44 private ITransactionManager trans { 44 45 get { return ServiceLocator.Instance.TransactionManager; } 45 46 } -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/HeartbeatManager.cs
r12012 r12468 25 25 using System.Threading; 26 26 using HeuristicLab.Services.Hive.DataAccess; 27 using HeuristicLab.Services.Hive.DataAccess.Interfaces; 27 28 using Heartbeat = HeuristicLab.Services.Hive.DataTransfer.Heartbeat; 28 29 … … 37 38 get { return ServiceLocator.Instance.TaskScheduler; } 38 39 } 39 private DataAccess.ITransactionManager trans {40 private ITransactionManager trans { 40 41 get { return ServiceLocator.Instance.TransactionManager; } 41 42 } -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/ServiceLocator.cs
r12441 r12468 21 21 22 22 using HeuristicLab.Services.Hive.DataAccess; 23 using HeuristicLab.Services.Hive.DataAccess.Interfaces; 24 using HeuristicLab.Services.Hive.DataAccess.Manager; 23 25 24 26 namespace HeuristicLab.Services.Hive { … … 32 34 } 33 35 set { instance = value; } 36 } 37 38 public IPersistenceManager PersistenceManager { 39 get { return new PersistenceManager(); } 34 40 } 35 41
Note: See TracChangeset
for help on using the changeset viewer.