Changeset 9557 for branches/HiveStatistics
- Timestamp:
- 05/29/13 15:17:03 (11 years ago)
- Location:
- branches/HiveStatistics/sources
- Files:
-
- 5 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml
r9546 r9557 116 116 <Column Name="Data" Type="System.Byte[]" DbType="VarBinary(MAX)" CanBeNull="false" UpdateCheck="Never" IsDelayLoaded="true" /> 117 117 <Column Name="LastUpdate" Type="System.DateTime" DbType="DateTime" CanBeNull="false" /> 118 <Column Name="DataSize" Type="System.Int64" DbType="BigInt NOT NULL" IsReadOnly="true" IsDbGenerated="true" CanBeNull="false" /> 118 119 <Association Name="Task_TaskData" Member="Task" Storage="_Job" ThisKey="TaskId" OtherKey="TaskId" Type="Task" IsForeignKey="true" /> 119 120 </Type> … … 212 213 <Table Name="[statistics].DimClient" Member="DimClients"> 213 214 <Type Name="DimClient"> 214 <Column Name="Id" AutoSync="Never"Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" IsDbGenerated="true" CanBeNull="false" />215 <Column Name="Id" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" IsDbGenerated="true" CanBeNull="false" /> 215 216 <Column Name="Name" Type="System.String" DbType="VarChar(MAX) NOT NULL" CanBeNull="false" /> 216 217 <Column Name="ResourceId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="false" /> … … 224 225 <Table Name="[statistics].FactTask" Member="FactTasks"> 225 226 <Type Name="FactTask"> 226 <Column Name="TaskId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="false" />227 <Column Name="TaskId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" CanBeNull="false" /> 227 228 <Column Name="TotalRuntime" Type="System.Double" DbType="Float NOT NULL" CanBeNull="false" /> 228 229 <Column Name="TotalWaitingTime" Type="System.Double" DbType="Float NOT NULL" CanBeNull="false" /> … … 232 233 <Column Name="CoresRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" /> 233 234 <Column Name="MemoryRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" /> 234 <Column Name="TaskSize" Type="System.Int 32" DbType="Int NOT NULL" CanBeNull="false" />235 <Column Name="ResultSize" Type="System.Int 32" DbType="Int" CanBeNull="true" />235 <Column Name="TaskSize" Type="System.Int64" DbType="BigInt NOT NULL" CanBeNull="false" /> 236 <Column Name="ResultSize" Type="System.Int64" DbType="BigInt" CanBeNull="true" /> 236 237 <Column Name="Priority" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" /> 237 238 <Column Name="LastClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" CanBeNull="false" /> … … 241 242 <Association Name="DimClient_FactTask" Member="DimClient" ThisKey="LastClientId" OtherKey="Id" Type="DimClient" IsForeignKey="true" /> 242 243 <Association Name="DimJob_FactTask" Member="DimJob" ThisKey="JobId" OtherKey="JobId" Type="DimJob" IsForeignKey="true" /> 243 <Association Name="DimTime_FactTask" Member="DimTime " ThisKey="EndTime" OtherKey="Time" Type="DimTime" IsForeignKey="true" />244 <Association Name="DimTime_FactTask1" Member="DimTime 1" ThisKey="StartTime" OtherKey="Time" Type="DimTime" IsForeignKey="true" />244 <Association Name="DimTime_FactTask" Member="DimTimeEnd" Storage="_DimTime" ThisKey="EndTime" OtherKey="Time" Type="DimTime" IsForeignKey="true" /> 245 <Association Name="DimTime_FactTask1" Member="DimTimeStart" Storage="_DimTime1" ThisKey="StartTime" OtherKey="Time" Type="DimTime" IsForeignKey="true" /> 245 246 </Type> 246 247 </Table> -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml.layout
r9541 r9557 76 76 </nodes> 77 77 </associationConnector> 78 <classShape Id="6bc13f26-f9a8-4597-b054-35be34190d12" absoluteBounds="4.125, 1, 2, 1. 26050048828125">78 <classShape Id="6bc13f26-f9a8-4597-b054-35be34190d12" absoluteBounds="4.125, 1, 2, 1.424847819010417"> 79 79 <DataClassMoniker Name="/HiveDataContext/TaskData" /> 80 80 <nestedChildShapes> 81 <elementListCompartment Id="a068522c-7974-4679-b356-e33c941c465b" absoluteBounds="4.14, 1.46, 1.9700000000000002, 0. 70050048828125" name="DataPropertiesCompartment" titleTextColor="Black" itemTextColor="Black" />81 <elementListCompartment Id="a068522c-7974-4679-b356-e33c941c465b" absoluteBounds="4.14, 1.46, 1.9700000000000002, 0.86484781901041663" name="DataPropertiesCompartment" titleTextColor="Black" itemTextColor="Black" /> 82 82 </nestedChildShapes> 83 83 </classShape> … … 121 121 </nestedChildShapes> 122 122 </classShape> 123 <associationConnector edgePoints="[(11.25 : 1.44547136382195); (11.0260416666667 : 1.44547136382195); (11.0260416666667 : 0.5625); (2.75 : 0.5625); (2.75 : 1)]" manuallyRouted="true" fixedFrom=" Caller" fixedTo="Algorithm">123 <associationConnector edgePoints="[(11.25 : 1.44547136382195); (11.0260416666667 : 1.44547136382195); (11.0260416666667 : 0.5625); (2.75 : 0.5625); (2.75 : 1)]" manuallyRouted="true" fixedFrom="NotFixed" fixedTo="NotFixed"> 124 124 <AssociationMoniker Name="/HiveDataContext/Resource/Resource_StateLog" /> 125 125 <nodes> … … 179 179 </nodes> 180 180 </associationConnector> 181 <associationConnector edgePoints="[(8.0192961430407 : 3.90397379557292); (8.0192961430407 : 5.04807657877604); (8.875 : 5.04807657877604)]" fixedFrom=" Caller" fixedTo="Algorithm">181 <associationConnector edgePoints="[(8.0192961430407 : 3.90397379557292); (8.0192961430407 : 5.04807657877604); (8.875 : 5.04807657877604)]" fixedFrom="NotFixed" fixedTo="NotFixed"> 182 182 <AssociationMoniker Name="/HiveDataContext/Task/Task_AssignedResource" /> 183 183 <nodes> … … 186 186 </nodes> 187 187 </associationConnector> 188 <associationConnector edgePoints="[(7.20145120722822 : 3.90397379557292); (7.20145120722822 : 5.875)]" fixedFrom=" Caller" fixedTo="Algorithm">188 <associationConnector edgePoints="[(7.20145120722822 : 3.90397379557292); (7.20145120722822 : 5.875)]" fixedFrom="NotFixed" fixedTo="NotFixed"> 189 189 <AssociationMoniker Name="/HiveDataContext/Task/Task_RequiredPlugin" /> 190 190 <nodes> … … 193 193 </nodes> 194 194 </associationConnector> 195 <associationConnector edgePoints="[(7.20923076923077 : 1); (7.20923076923077 : 0.770833333333333); (7.80461538461539 : 0.770833333333333); (7.80461538461539 : 1)]" manuallyRouted="true" fixedFrom=" Algorithm" fixedTo="Algorithm">195 <associationConnector edgePoints="[(7.20923076923077 : 1); (7.20923076923077 : 0.770833333333333); (7.80461538461539 : 0.770833333333333); (7.80461538461539 : 1)]" manuallyRouted="true" fixedFrom="NotFixed" fixedTo="NotFixed"> 196 196 <AssociationMoniker Name="/HiveDataContext/Task/Task_Task" /> 197 197 <nodes> … … 200 200 </nodes> 201 201 </associationConnector> 202 <associationConnector edgePoints="[(6.5 : 1. 69314697265625); (6.125 : 1.69314697265625)]" fixedFrom="NotFixed" fixedTo="NotFixed">202 <associationConnector edgePoints="[(6.5 : 1.71242390950521); (6.125 : 1.71242390950521)]" fixedFrom="NotFixed" fixedTo="NotFixed"> 203 203 <AssociationMoniker Name="/HiveDataContext/Task/Task_TaskData" /> 204 204 <nodes> … … 207 207 </nodes> 208 208 </associationConnector> 209 <associationConnector edgePoints="[(6.5 : 2.5 0564697265625); (3.75 : 2.50564697265625)]" fixedFrom="NotFixed" fixedTo="NotFixed">209 <associationConnector edgePoints="[(6.5 : 2.52492390950521); (3.75 : 2.52492390950521)]" fixedFrom="NotFixed" fixedTo="NotFixed"> 210 210 <AssociationMoniker Name="/HiveDataContext/Task/Task_StateLog" /> 211 211 <nodes> -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs
r9546 r9557 2731 2731 private System.DateTime _LastUpdate; 2732 2732 2733 private long _DataSize = default(long); 2734 2733 2735 private EntityRef<Task> _Job; 2734 2736 … … 2812 2814 this.OnLastUpdateChanged(); 2813 2815 } 2816 } 2817 } 2818 2819 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_DataSize", AutoSync=AutoSync.Always, DbType="BigInt NOT NULL", IsDbGenerated=true, UpdateCheck=UpdateCheck.Never)] 2820 public long DataSize 2821 { 2822 get 2823 { 2824 return this._DataSize; 2814 2825 } 2815 2826 } … … 4648 4659 } 4649 4660 4650 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_Id", DbType="UniqueIdentifier NOT NULL", IsPrimaryKey=true, IsDbGenerated=true)]4661 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_Id", AutoSync=AutoSync.OnInsert, DbType="UniqueIdentifier NOT NULL", IsPrimaryKey=true, IsDbGenerated=true)] 4651 4662 public System.Guid Id 4652 4663 { … … 4861 4872 private int _MemoryRequired; 4862 4873 4863 private int_TaskSize;4864 4865 private System.Nullable< int> _ResultSize;4874 private long _TaskSize; 4875 4876 private System.Nullable<long> _ResultSize; 4866 4877 4867 4878 private int _Priority; … … 4903 4914 partial void OnMemoryRequiredChanging(int value); 4904 4915 partial void OnMemoryRequiredChanged(); 4905 partial void OnTaskSizeChanging( intvalue);4916 partial void OnTaskSizeChanging(long value); 4906 4917 partial void OnTaskSizeChanged(); 4907 partial void OnResultSizeChanging(System.Nullable< int> value);4918 partial void OnResultSizeChanging(System.Nullable<long> value); 4908 4919 partial void OnResultSizeChanged(); 4909 4920 partial void OnPriorityChanging(int value); … … 4928 4939 } 4929 4940 4930 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_TaskId", DbType="UniqueIdentifier NOT NULL" )]4941 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_TaskId", DbType="UniqueIdentifier NOT NULL", IsPrimaryKey=true)] 4931 4942 public System.Guid TaskId 4932 4943 { … … 5088 5099 } 5089 5100 5090 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_TaskSize", DbType=" Int NOT NULL")]5091 public intTaskSize5101 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_TaskSize", DbType="BigInt NOT NULL")] 5102 public long TaskSize 5092 5103 { 5093 5104 get … … 5108 5119 } 5109 5120 5110 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_ResultSize", DbType=" Int")]5111 public System.Nullable< int> ResultSize5121 [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_ResultSize", DbType="BigInt")] 5122 public System.Nullable<long> ResultSize 5112 5123 { 5113 5124 get … … 5313 5324 5314 5325 [global::System.Data.Linq.Mapping.AssociationAttribute(Name="DimTime_FactTask", Storage="_DimTime", ThisKey="EndTime", OtherKey="Time", IsForeignKey=true)] 5315 public DimTime DimTime 5326 public DimTime DimTimeEnd 5316 5327 { 5317 5328 get … … 5341 5352 this._EndTime = default(System.DateTime); 5342 5353 } 5343 this.SendPropertyChanged("DimTime ");5354 this.SendPropertyChanged("DimTimeEnd"); 5344 5355 } 5345 5356 } … … 5347 5358 5348 5359 [global::System.Data.Linq.Mapping.AssociationAttribute(Name="DimTime_FactTask1", Storage="_DimTime1", ThisKey="StartTime", OtherKey="Time", IsForeignKey=true)] 5349 public DimTime DimTime 15360 public DimTime DimTimeStart 5350 5361 { 5351 5362 get … … 5375 5386 this._StartTime = default(System.DateTime); 5376 5387 } 5377 this.SendPropertyChanged("DimTime 1");5388 this.SendPropertyChanged("DimTimeStart"); 5378 5389 } 5379 5390 } … … 5771 5782 { 5772 5783 this.SendPropertyChanging(); 5773 entity.DimTime = this;5784 entity.DimTimeEnd = this; 5774 5785 } 5775 5786 … … 5777 5788 { 5778 5789 this.SendPropertyChanging(); 5779 entity.DimTime = null;5790 entity.DimTimeEnd = null; 5780 5791 } 5781 5792 … … 5783 5794 { 5784 5795 this.SendPropertyChanging(); 5785 entity.DimTime 1= this;5796 entity.DimTimeStart = this; 5786 5797 } 5787 5798 … … 5789 5800 { 5790 5801 this.SendPropertyChanging(); 5791 entity.DimTime 1= null;5802 entity.DimTimeStart = null; 5792 5803 } 5793 5804 -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql
r9546 r9557 96 96 [Data] VarBinary(MAX) Filestream NOT NULL, 97 97 [LastUpdate] DateTime NOT NULL, 98 [DataSize] AS (datalength([Data])), 98 99 CONSTRAINT [PK_dbo.TaskData] PRIMARY KEY ([TaskId]) 99 100 ) … … 260 261 [CoresRequired] INT NOT NULL, 261 262 [MemoryRequired] INT NOT NULL, 262 [TaskSize] INTNOT NULL,263 [ResultSize] INTNULL,263 [TaskSize] BIGINT NOT NULL, 264 [ResultSize] BIGINT NULL, 264 265 [Priority] INT NOT NULL, 265 266 [LastClientId] UNIQUEIDENTIFIER NOT NULL, … … 267 268 [StartTime] DATETIME NOT NULL, 268 269 [EndTime] DATETIME NOT NULL, 269 CONSTRAINT [PK_FactTask] PRIMARY KEY CLUSTERED ([ LastClientId] ASC, [JobId] ASC, [StartTime] ASC, [EndTime] ASC),270 CONSTRAINT [PK_FactTask] PRIMARY KEY CLUSTERED ([TaskId] ASC, [LastClientId] ASC, [JobId] ASC, [StartTime] ASC, [EndTime] ASC), 270 271 CONSTRAINT [FK_FactTask_DimClient] FOREIGN KEY ([LastClientId]) REFERENCES [statistics].[DimClient] ([Id]), 271 272 CONSTRAINT [FK_FactTask_DimJob] FOREIGN KEY ([JobId]) REFERENCES [statistics].[DimJob] ([JobId]), … … 273 274 CONSTRAINT [FK_FactTask_DimTimeEnd] FOREIGN KEY ([EndTime]) REFERENCES [statistics].[DimTime] ([Time]) 274 275 ); 276 277 INSERT INTO [HeuristicLab.Hive-3.3].[statistics].[DimClient] ([Id] ,[Name] ,[ResourceId]) 278 VALUES ('00000000-0000-0000-0000-000000000000' ,'NULL' ,'00000000-0000-0000-0000-000000000000') -
branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs
r9546 r9557 14 14 public void GenerateStatistics() { 15 15 using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString)) 16 using (var transaction = new TransactionScope()) { 16 using (var transaction = new TransactionScope(TransactionScopeOption.Required, new TransactionOptions { IsolationLevel = IsolationLevel.ReadCommitted })) { 17 17 18 var newTime = UpdateDimensionTables(db); 18 19 db.SubmitChanges(); … … 30 31 var newTime = UpdateTime(db); 31 32 // Update other tables out of sync with time dimension? 33 UpdateUsers(db); 32 34 UpdateJobs(db); 33 UpdateUsers(db);34 35 UpdateClients(db); 35 36 … … 38 39 39 40 private DimTime UpdateTime(HiveDataContext db) { 40 var lastUpdate =41 var lastUpdateTime = 41 42 (from t in db.DimTimes 42 43 orderby t.Time descending … … 46 47 var now = DateTime.Now; 47 48 DimTime newTime = null; 48 if (lastUpdate == default(DateTime) || lastUpdate + SmallestTimeSpan < now) { 49 50 if (lastUpdateTime == default(DateTime) || lastUpdateTime + SmallestTimeSpan < now) { 49 51 newTime = new DimTime { 50 52 Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute - now.Minute % SmallestTimeSpan.Minutes, 0), … … 58 60 59 61 return newTime; 62 } 63 64 private void UpdateUsers(HiveDataContext db) { 65 var newUsers = 66 from u in db.Resources.Where(x => x.OwnerUserId != null).Select(x => x.OwnerUserId.Value).Union(db.Jobs.Select(x => x.OwnerUserId)) 67 where !db.DimUsers.Select(x => x.UserId).Contains(u) 68 select u; 69 70 var newDimUsers = 71 from u in newUsers.ToList() 72 select new DimUser { 73 UserId = u, 74 Name = userManager.GetUserById(u).UserName 75 }; 76 77 db.DimUsers.InsertAllOnSubmit(newDimUsers); 78 79 // insert NULL-User 80 if (!db.DimUsers.Any(x => x.UserId == Guid.Empty)) { 81 db.DimUsers.InsertOnSubmit(new DimUser { UserId = Guid.Empty, Name = "NULL" }); 82 } 60 83 } 61 84 … … 76 99 77 100 db.DimJobs.InsertAllOnSubmit(newDimJobs); 78 } 79 80 private void UpdateUsers(HiveDataContext db) { 81 var newUsers = 82 from u in db.Resources.Where(x => x.OwnerUserId != null).Select(x => x.OwnerUserId.Value).Union(db.Jobs.Select(x => x.OwnerUserId)) 83 where !db.DimUsers.Select(x => x.UserId).Contains(u) 84 select u; 85 86 var newDimUsers = 87 from u in newUsers.ToList() 88 select new DimUser { 89 UserId = u, 90 Name = userManager.GetUserById(u).UserName 91 }; 92 93 db.DimUsers.InsertAllOnSubmit(newDimUsers); 94 95 // insert NULL-User 96 if (!db.DimUsers.Any(x => x.UserId == Guid.Empty)) { 97 db.DimUsers.InsertOnSubmit(new DimUser { UserId = Guid.Empty, Name = "NULL" }); 101 102 // insert NULL-Job 103 if (!db.DimJobs.Any(x => x.JobId == Guid.Empty)) { 104 db.DimJobs.InsertOnSubmit(new DimJob { JobId = Guid.Empty, JobName = "NULL", UserId = Guid.Empty, UserName = "NULL" }); 98 105 } 99 106 } … … 143 150 private void UpdateFactTables(DimTime newTime, HiveDataContext db) { 144 151 UpdateClientInfoFacts(newTime, db); 145 //UpdateTaskFacts(newTime, db);152 UpdateTaskFacts(newTime, db); 146 153 } 147 154 … … 213 220 : selector(lastFact); 214 221 } 222 223 private void UpdateTaskFacts(DimTime newTime, HiveDataContext db) { 224 var lastFacts = 225 from f in db.FactTasks 226 group f by f.TaskId into grpFacts 227 select grpFacts.OrderByDescending(x => x.EndTime).First(); 228 229 var tasksWithLastFact = 230 from t in db.Tasks 231 join lf in lastFacts on t.TaskId equals lf.TaskId into tLfJoin 232 from lWithLf in tLfJoin.DefaultIfEmpty() 233 where !t.IsParentTask 234 select new { Task = t, LastFact = tLfJoin.FirstOrDefault() }; 235 236 var tasksWithNewStateLogs = 237 from sl in db.StateLogs 238 join twlf in tasksWithLastFact on sl.TaskId equals twlf.Task.TaskId 239 where twlf.LastFact == null || (sl.DateTime > twlf.LastFact.EndTime && sl.DateTime < newTime.Time) 240 group sl by twlf.Task into factGroup 241 select new { Task = factGroup.Key, StateLogs = factGroup.OrderBy(x => x.DateTime) }; 242 243 var tasksWithLastFactsAndStateLogs = 244 from twsl in tasksWithNewStateLogs 245 join twlf in tasksWithLastFact on twsl.Task equals twlf.Task 246 select new { twlf.Task, twlf.LastFact, twsl.StateLogs }; 247 248 var newTaskFacts = 249 from t in tasksWithLastFactsAndStateLogs.ToList() 250 select new FactTask { 251 TaskId = t.Task.TaskId, 252 JobId = t.Task.JobId, 253 DimTimeStart = t.LastFact.DimTimeStart, 254 DimTimeEnd = newTime, 255 // ... 256 }; 257 258 db.FactTasks.InsertAllOnSubmit(newTaskFacts); 259 } 215 260 } 216 261 }
Note: See TracChangeset
for help on using the changeset viewer.