Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
06/18/15 11:58:58 (9 years ago)
Author:
dglaser
Message:

#2388:
HeuristicLab.Services.Hive.DataAccess-3.3:

  • Added PersistenceManager with corresponding daos
  • Updated SQL Scripts
  • Fixed folder structure (interfaces, manager)
  • Removed duplicated IHiveDao and HiveDao (the HiveDao/OptimizedHiveDao that is actually used is located in HeuristicLab.Services.Hive)

HeuristicLab.Service.Hive-3.3:

  • Added PersistenceManager to the ServiceLocater
  • Updated and improved the HiveStatisticsGenerator
  • Updated HiveJanitor
Location:
branches/HiveStatistics/sources
Files:
31 added
6 deleted
14 edited

Legend:

Unmodified
Added
Removed

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HeuristicLab.Services.Hive.DataAccess-3.3.csproj

    r11623 r12468  
    104104  <ItemGroup>
    105105    <None Include="Plugin.cs.frame" />
     106    <Compile Include="Daos\AssignedResourceDao.cs" />
     107    <Compile Include="Daos\DowntimeDao.cs" />
     108    <Compile Include="Daos\GenericDao.cs" />
     109    <Compile Include="Daos\HiveStatistics\DimClientDao.cs" />
     110    <Compile Include="Daos\HiveStatistics\DimJobDao.cs" />
     111    <Compile Include="Daos\HiveStatistics\DimTimeDao.cs" />
     112    <Compile Include="Daos\HiveStatistics\DimUserDao.cs" />
     113    <Compile Include="Daos\HiveStatistics\FactClientInfoDao.cs" />
     114    <Compile Include="Daos\HiveStatistics\FactTaskDao.cs" />
     115    <Compile Include="Daos\JobDao.cs" />
     116    <Compile Include="Daos\JobPermissionDao.cs" />
     117    <Compile Include="Daos\LifecycleDao.cs" />
     118    <Compile Include="Daos\PluginDao.cs" />
     119    <Compile Include="Daos\PluginDataDao.cs" />
     120    <Compile Include="Daos\RequiredPluginDao.cs" />
     121    <Compile Include="Daos\ResourceDao.cs" />
     122    <Compile Include="Daos\ResourcePermissionDao.cs" />
     123    <Compile Include="Daos\SlaveDao.cs" />
     124    <Compile Include="Daos\SlaveGroupDao.cs" />
     125    <Compile Include="Daos\StateLogDao.cs" />
     126    <Compile Include="Daos\TaskDao.cs" />
     127    <Compile Include="Daos\TaskDataDao.cs" />
     128    <Compile Include="Daos\UserPriorityDao.cs" />
     129    <Compile Include="Interfaces\IGenericDao.cs" />
    106130    <Compile Include="Enums\Command.cs" />
    107131    <Compile Include="Enums\DowntimeType.cs" />
    108132    <Compile Include="Enums\CpuArchitecture.cs" />
     133    <Compile Include="Interfaces\IPersistenceManager.cs" />
     134    <Compile Include="Manager\PersistenceManager.cs" />
    109135    <Compile Include="Plugin.cs" />
    110136    <Compile Include="HiveDataContext.cs">
    … …  
    116142      <DependentUpon>HiveDataContext.dbml</DependentUpon>
    117143    </Compile>
    118     <Compile Include="ITransactionManager.cs" />
     144    <Compile Include="Interfaces\ITransactionManager.cs" />
    119145    <Compile Include="Enums\TaskState.cs" />
    120146    <Compile Include="Logger.cs" />
    … …  
    128154    </Compile>
    129155    <Compile Include="Enums\SlaveState.cs" />
    130     <Compile Include="TransactionManager.cs" />
     156    <Compile Include="Manager\TransactionManager.cs" />
    131157  </ItemGroup>
    132158  <ItemGroup>
    … …  
    158184    <Content Include="SQL Scripts\Prepare Hive Database.sql" />
    159185  </ItemGroup>
     186  <ItemGroup />
    160187  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
    161188  <PropertyGroup>

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml

    r12441 r12468  
    11<?xml version="1.0" encoding="utf-8"?><Database Name="HeuristicLab.Hive-3.3" Class="HiveDataContext" xmlns="http://schemas.microsoft.com/linqtosql/dbml/2007">
    2   <Connection Mode="AppSettings" ConnectionString="Data Source=.\SQLEXPRESS;Initial Catalog=HeuristicLab.Hive-3.3;Integrated Security=True" SettingsObjectName="HeuristicLab.Services.Hive.DataAccess.Settings" SettingsPropertyName="HeuristicLab_Hive_LinqConnectionString" Provider="System.Data.SqlClient" />
     2  <Connection Mode="AppSettings" ConnectionString="Data Source=localhost;Initial Catalog=HeuristicLab.Hive-3.3;Integrated Security=True" SettingsObjectName="HeuristicLab.Services.Hive.DataAccess.Settings" SettingsPropertyName="HeuristicLab_Hive_LinqConnectionString" Provider="System.Data.SqlClient" />
    33  <Table Name="dbo.AssignedResources" Member="AssignedResources">
    44    <Type Name="AssignedResource">
    … …  
    233233      <Column Name="MemoryRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
    234234      <Column Name="Priority" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
    235       <Column Name="LastClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="false" />
     235      <Column Name="LastClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="true" />
    236236      <Column Name="JobId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="false" />
    237237      <Column Name="StartTime" Type="System.DateTime" DbType="DateTime NOT NULL" CanBeNull="false" />

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml.layout

    r12441 r12468  
    297297      </nodes>
    298298    </associationConnector>
    299     <associationConnector edgePoints="[(5.02343625 : 10.2708968098958); (5.02343625 : 14.7227579166667); (6.875 : 14.7227579166667)]" fixedFrom="NotFixed" fixedTo="NotFixed">
     299    <associationConnector edgePoints="[(5.02343625 : 10.2708968098958); (5.02343625 : 14.7227579166667); (5.90885291666667 : 14.7227579166667 : JumpStart); (6.07551958333333 : 14.7227579166667 : JumpEnd); (6.875 : 14.7227579166667)]" fixedFrom="NotFixed" fixedTo="NotFixed">
    300300      <AssociationMoniker Name="/HiveDataContext/DimTime/DimTime_FactTask" />
    301301      <nodes>

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs

    r12441 r12468  
    33// <auto-generated>
    44//     This code was generated by a tool.
    5 //     Runtime Version:4.0.30319.34014
     5//     Runtime Version:4.0.30319.34209
    66//
    77//     Changes to this file may cause incorrect behavior and will be lost if
    … …  
    48634863    private int _Priority;
    48644864   
    4865     private System.Guid _LastClientId;
     4865    private System.Nullable<System.Guid> _LastClientId;
    48664866   
    48674867    private System.Guid _JobId;
    … …  
    49054905    partial void OnPriorityChanging(int value);
    49064906    partial void OnPriorityChanged();
    4907     partial void OnLastClientIdChanging(System.Guid value);
     4907    partial void OnLastClientIdChanging(System.Nullable<System.Guid> value);
    49084908    partial void OnLastClientIdChanged();
    49094909    partial void OnJobIdChanging(System.Guid value);
    … …  
    51095109   
    51105110    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_LastClientId", DbType="UniqueIdentifier NOT NULL")]
    5111     public System.Guid LastClientId
     5111    public System.Nullable<System.Guid> LastClientId
    51125112    {
    51135113      get
    … …  
    52715271          else
    52725272          {
    5273             this._LastClientId = default(System.Guid);
     5273            this._LastClientId = default(Nullable<System.Guid>);
    52745274          }
    52755275          this.SendPropertyChanged("DimClient");

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql

    r12441 r12468  
    239239    [TotalMemory]           INT              NOT NULL,
    240240    [CpuUtilization]        FLOAT (53)       NOT NULL,
    241   [SlaveState]      VarChar(15)    NOT NULL,
     241    [SlaveState]      VarChar(15)    NOT NULL,
    242242    [TotalTimeIdle]         FLOAT (53)       NOT NULL,
    243243    [TotalTimeCalculating]  FLOAT (53)       NOT NULL,
    … …  
    260260    [MemoryRequired]     INT              NOT NULL,
    261261    [Priority]           INT              NOT NULL,
    262     [LastClientId]       UNIQUEIDENTIFIER NOT NULL,
     262    [LastClientId]       UNIQUEIDENTIFIER NULL,
    263263    [JobId]              UNIQUEIDENTIFIER NOT NULL,
    264264    [StartTime]          DATETIME         NOT NULL,
    265265    [EndTime]            DATETIME         NULL,
    266   [TaskState]          VARCHAR (30)     NOT NULL,
    267   [Exception]      VARCHAR (MAX)    NULL,
     266    [TaskState]          VARCHAR (30)     NOT NULL,
     267    [Exception]      VARCHAR (MAX)    NULL,
    268268    CONSTRAINT [PK_FactTask] PRIMARY KEY CLUSTERED ([TaskId] ASC),
    269269    CONSTRAINT [FK_FactTask_DimClient] FOREIGN KEY ([LastClientId]) REFERENCES [statistics].[DimClient] ([Id]),
    … …  
    273273);
    274274
    275 INSERT INTO [HeuristicLab.Hive-3.3].[statistics].[DimClient] ([Id] ,[Name] ,[ResourceId])
    276 VALUES ('00000000-0000-0000-0000-000000000000' ,'NULL' ,'00000000-0000-0000-0000-000000000000')
     275/* dummy for nullable userIds in FactClientInfo */
     276INSERT INTO [statistics].[DimUser] ([UserId], [Name])
     277VALUES ('00000000-0000-0000-0000-000000000000', 'NULL');

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.Web/Hive-3.3/Status.aspx.cs

    r12146 r12468  
    8484
    8585    overallCpuUtilizationLabel.Text = (onlineSlaves.Any() ? Math.Round(onlineSlaves.Average(s => s.CpuUtilization), 2).ToString() : "0.0") + " %";
    86     cpuUtilizationLabel.Text = (onlineSlaves.Any() && onlineSlaves.Any(x => x.IsAllowedToCalculate) ? Math.Round(onlineSlaves.Where(x => x.IsAllowedToCalculate).Average(s => s.CpuUtilization), 2).ToString() : "0.0") + " %";
     86    cpuUtilizationLabel.Text = (onlineSlaves.Any(x => x.IsAllowedToCalculate) ? Math.Round(onlineSlaves.Where(x => x.IsAllowedToCalculate).Average(s => s.CpuUtilization), 2).ToString() : "0.0") + " %";
    8787
    8888    DT.Statistics[] stats = new DT.Statistics[0];

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveDao.cs

    r12146 r12468  
    891891    public List<DT.UserStatistics> GetUserStatistics() {
    892892      using (var db = CreateContext()) {
     893        db.Log = Console.Out;
    893894        var userStats = new Dictionary<Guid, DT.UserStatistics>();
    894895

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveJanitor.cs

    r12441 r12468  
    2323using System.Threading;
    2424using HeuristicLab.Services.Hive.DataAccess;
     25using HeuristicLab.Services.Hive.DataAccess.Interfaces;
    2526
    2627namespace HeuristicLab.Services.Hive {
    … …  
    3031    private AutoResetEvent generateStatisticsWaitHandle;
    3132
    32     private DataAccess.ITransactionManager trans {
     33    private ITransactionManager trans {
    3334      get { return ServiceLocator.Instance.TransactionManager; }
    3435    }
    … …  
    5556      stop = true;
    5657      cleanupWaitHandle.Set();
     58      generateStatisticsWaitHandle.Set();
    5759    }
    5860

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveService.cs

    r12012 r12468  
    2424using System.Linq;
    2525using System.ServiceModel;
     26using HeuristicLab.Services.Hive.DataAccess.Interfaces;
    2627using HeuristicLab.Services.Hive.DataTransfer;
    2728using HeuristicLab.Services.Hive.ServiceContracts;
    … …  
    5152      get { return ServiceLocator.Instance.AuthorizationManager; }
    5253    }
    53     private DataAccess.ITransactionManager trans {
     54    private ITransactionManager trans {
    5455      get { return ServiceLocator.Instance.TransactionManager; }
    5556    }

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

    r12441 r12468  
    1 using System;
     1#region License Information
     2/* HeuristicLab
     3 * Copyright (C) 2002-2015 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
     4 *
     5 * This file is part of HeuristicLab.
     6 *
     7 * HeuristicLab is free software: you can redistribute it and/or modify
     8 * it under the terms of the GNU General Public License as published by
     9 * the Free Software Foundation, either version 3 of the License, or
     10 * (at your option) any later version.
     11 *
     12 * HeuristicLab is distributed in the hope that it will be useful,
     13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
     14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
     15 * GNU General Public License for more details.
     16 *
     17 * You should have received a copy of the GNU General Public License
     18 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
     19 */
     20#endregion
     21
     22using System;
    223using System.Collections.Generic;
    324using System.Data.Linq;
    425using System.Linq;
    5 using System.Transactions;
    6 using HeuristicLab.Services.Access;
    726using HeuristicLab.Services.Hive.DataAccess;
     27using HeuristicLab.Services.Hive.DataAccess.Manager;
    828
    929namespace HeuristicLab.Services.Hive {
    1030  public class HiveStatisticsGenerator : IStatisticsGenerator {
    11     private IUserManager userManager { get { return ServiceLocator.Instance.UserManager; } }
    12 
     31
     32    private const string UnknownUserName = "Unknown";
    1333    private static readonly TimeSpan SmallestTimeSpan = new TimeSpan(0, 5, 0);
    14     private static readonly string UnknownUserName = "Unknown";
    1534
    1635    public void GenerateStatistics() {
    17       using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
    18       using (var transaction = new TransactionScope(TransactionScopeOption.Required, new TransactionOptions { IsolationLevel = IsolationLevel.ReadCommitted })) {
    19 
    20         var newTime = UpdateDimensionTables(db);
    21         db.SubmitChanges();
    22 
    23         if (newTime != null) {
    24           UpdateFactTables(newTime, db);
    25           try {
    26             db.SubmitChanges();
     36      using (var pm = new PersistenceManager()) {
     37        pm.UseTransaction(() => {
     38          UpdateDimUserTable(pm);
     39          UpdateDimJobTable(pm);
     40          UpdateDimClients(pm);
     41          pm.SubmitChanges();
     42        });
     43
     44        pm.UseTransaction(() => {
     45          var newTime = UpdateDimTimeTable(pm);
     46          if (newTime != null) {
     47            pm.SubmitChanges();
     48            UpdateFactClientInfoTable(newTime, pm);
     49            UpdateTaskFacts(newTime, pm);
     50            try {
     51              pm.SubmitChanges();
     52            }
     53            catch (DuplicateKeyException e) {
     54              var logger = LogFactory.GetLogger(typeof(HiveStatisticsGenerator).Namespace);
     55              logger.Log(string.Format(
     56                @"Propable change from summertime to wintertime, resulting in overlapping times.
     57                          On wintertime to summertime change, slave timeouts and a fact gap will occur.
     58                          Exception Details: {0}", e));
     59            }
    2760          }
    28           catch (DuplicateKeyException) {
    29             // Propable change from summertime to wintertime, resulting in overlapping times
    30             // On wintertime to summertime change, slave timeouts and a fact gap will occur
    31           }
     61        });
     62      }
     63    }
     64
     65    private DimTime UpdateDimTimeTable(PersistenceManager pm) {
     66      var dimTimeDao = pm.DimTimeDao;
     67      var now = DateTime.Now;
     68      var timeEntry = new DimTime {
     69        Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute, 0),
     70        Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
     71        Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
     72        Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
     73        Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
     74      };
     75      // using SaveOrAttach instead of save because the table can only be updated once every minute
     76      // (pk restriction) save would cause a duplicate key exception if the method is accidentally
     77      // called more than once every minute (e.g. restarting the service within a minute)
     78      return dimTimeDao.SaveOrAttach(timeEntry);
     79    }
     80
     81    private void UpdateDimUserTable(PersistenceManager pm) {
     82      var dimUserDao = pm.DimUserDao;
     83      var resourceDao = pm.ResourceDao;
     84      var jobDao = pm.JobDao;
     85      var existingUserIds = dimUserDao.GetAll().Select(x => x.UserId);
     86      var vaildResourceOwnerIds = resourceDao.GetResourcesWithValidOwner().Select(x => x.OwnerUserId.Value);
     87      var jobOwnerIds = jobDao.GetAll().Select(x => x.OwnerUserId);
     88      var newUserIds = vaildResourceOwnerIds
     89        .Union(jobOwnerIds)
     90        .Where(id => !existingUserIds.Contains(id))
     91        .ToList();
     92      dimUserDao.Save(newUserIds.Select(x => new DimUser {
     93        UserId = x,
     94        Name = GetUserName(x)
     95      }));
     96    }
     97
     98    private void UpdateDimJobTable(PersistenceManager pm) {
     99      var dimJobDao = pm.DimJobDao;
     100      var jobDao = pm.JobDao;
     101      var dimJobIds = dimJobDao.GetAll().Select(x => x.JobId);
     102      var newJobs = jobDao.GetAll()
     103        .Where(x => !dimJobIds.Contains(x.JobId))
     104        .Select(x => new {
     105          JobId = x.JobId,
     106          UserId = x.OwnerUserId,
     107          JobName = x.Name ?? string.Empty,
     108        })
     109        .ToList();
     110      dimJobDao.Save(newJobs.Select(x => new DimJob {
     111        JobId = x.JobId,
     112        JobName = x.JobName,
     113        UserId = x.UserId,
     114        UserName = GetUserName(x.UserId)
     115      }));
     116    }
     117
     118    private void UpdateDimClients(PersistenceManager pm) {
     119      var dimClientDao = pm.DimClientDao;
     120      var slaveDao = pm.SlaveDao;
     121      var slaves = slaveDao.GetAll();
     122      var recentlyAddedClients = dimClientDao.GetRecentlyAddedClients();
     123      var slaveIds = slaves.Select(x => x.ResourceId);
     124
     125      var removedClientIds = recentlyAddedClients
     126        .Where(x => !slaveIds.Contains(x.ResourceId))
     127        .Select(x => x.Id);
     128      var modifiedClients =
     129        from slave in slaves
     130        join client in recentlyAddedClients on slave.ResourceId equals client.ResourceId
     131        where (slave.Name != client.Name
     132               || slave.ParentResourceId != client.ResourceGroupId
     133               || slave.ParentResource.ParentResourceId != client.ResourceGroup2Id)
     134        select new {
     135          SlaveId = slave.ResourceId,
     136          ClientId = client.Id
     137        };
     138      var clientIds = dimClientDao.GetAll().Select(x => x.ResourceId);
     139      var modifiedClientIds = modifiedClients.Select(x => x.SlaveId);
     140      var newClients = slaves
     141        .Where(x => !clientIds.Contains(x.ResourceId)
     142                    || modifiedClientIds.Contains(x.ResourceId))
     143        .Select(x => new {
     144          x.ResourceId,
     145          x.Name,
     146          ResourceGroupId = x.ParentResourceId,
     147          ResourceGroup2Id = x.ParentResource.ParentResourceId
     148        })
     149        .ToList();
     150
     151      var clientsToUpdate = removedClientIds.Union(modifiedClients.Select(x => x.ClientId));
     152      dimClientDao.UpdateExpirationTime(clientsToUpdate, DateTime.Now);
     153      dimClientDao.Save(newClients.Select(x => new DimClient {
     154        ResourceId = x.ResourceId,
     155        Name = x.Name,
     156        ExpirationTime = null,
     157        ResourceGroupId = x.ResourceGroupId,
     158        ResourceGroup2Id = x.ResourceGroup2Id
     159      }));
     160    }
     161
     162    private void UpdateFactClientInfoTable(DimTime newTime, PersistenceManager pm) {
     163      var factClientInfoDao = pm.FactClientInfoDao;
     164      var slaveDao = pm.SlaveDao;
     165      var dimClientDao = pm.DimClientDao;
     166
     167      var newRawFactInfos =
     168        from s in slaveDao.GetAll()
     169        join c in dimClientDao.GetRecentlyAddedClients() on s.ResourceId equals c.ResourceId
     170        join lcf in factClientInfoDao.GetLastUpdateTimestamps() on c.ResourceId equals lcf.ResourceId into joinCf
     171        from cf in joinCf.DefaultIfEmpty()
     172        select new {
     173          ClientId = c.Id,
     174          UserId = s.OwnerUserId ?? Guid.Empty,
     175          TotalCores = s.Cores ?? 0,
     176          FreeCores = s.FreeCores ?? 0,
     177          TotalMemory = s.Memory ?? 0,
     178          FreeMemory = s.FreeMemory ?? 0,
     179          CpuUtilization = s.CpuUtilization,
     180          SlaveState = s.SlaveState,
     181          IsAllowedToCalculate = s.IsAllowedToCalculate,
     182          LastFactTimestamp = cf.Timestamp
     183        };
     184
     185      factClientInfoDao.Save(
     186        from x in newRawFactInfos.ToList()
     187        let duration = x.LastFactTimestamp != null
     188                       ? (newTime.Time - (DateTime)x.LastFactTimestamp).TotalMinutes
     189                       : SmallestTimeSpan.TotalMinutes
     190        select new FactClientInfo {
     191          ClientId = x.ClientId,
     192          DimTime = newTime,
     193          UserId = x.UserId,
     194          NumUsedCores = x.TotalCores - x.FreeCores,
     195          NumTotalCores = x.TotalMemory,
     196          UsedMemory = x.TotalMemory - x.FreeMemory,
     197          TotalMemory = x.TotalMemory,
     198          CpuUtilization = x.CpuUtilization,
     199          SlaveState = x.SlaveState,
     200          TotalTimeIdle = x.SlaveState == SlaveState.Idle && x.IsAllowedToCalculate ? duration : 0.0,
     201          TotalTimeCalculating = x.SlaveState == SlaveState.Calculating ? duration : 0.0,
     202          TotalTimeUnavailable = x.SlaveState == SlaveState.Idle && !x.IsAllowedToCalculate ? duration : 0.0,
     203          TotalTimeOffline = x.SlaveState == SlaveState.Offline ? duration : 0.0
    32204        }
    33 
    34         transaction.Complete();
    35       }
    36     }
    37 
    38     private DimTime UpdateDimensionTables(HiveDataContext db) {
    39       var newTime = UpdateTime(db);
    40 
    41       UpdateUsers(db);
    42       UpdateJobs(db);
    43       UpdateClients(db);
    44 
    45       return newTime;
    46     }
    47 
    48     private DimTime UpdateTime(HiveDataContext db) {
    49       var lastUpdateTime =
    50         (from t in db.DimTimes
    51          orderby t.Time descending
    52          select t.Time)
    53         .FirstOrDefault();
    54 
    55       var now = DateTime.Now;
    56       DimTime newTime = null;
    57 
    58       if (lastUpdateTime == default(DateTime) || lastUpdateTime + SmallestTimeSpan < now) {
    59         newTime = new DimTime {
    60           Time = new DateTime(now.Year, now.Month, now.Day, now.Hour, now.Minute - now.Minute % SmallestTimeSpan.Minutes, 0),
    61           Hour = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0),
    62           Day = new DateTime(now.Year, now.Month, now.Day, 0, 0, 0),
    63           Month = new DateTime(now.Year, now.Month, 1, 0, 0, 0),
    64           Year = new DateTime(now.Year, 1, 1, 0, 0, 0)
    65         };
    66         db.DimTimes.InsertOnSubmit(newTime);
    67       }
    68 
    69       return newTime;
    70     }
    71 
    72     private void UpdateUsers(HiveDataContext db) {
    73       var newUsers =
    74         from u in db.Resources.Where(x => x.OwnerUserId != null).Select(x => x.OwnerUserId.Value).Union(db.Jobs.Select(x => x.OwnerUserId))
    75         where !db.DimUsers.Select(x => x.UserId).Contains(u)
    76         select u;
    77 
    78 
    79       var newDimUsers =
    80      from u in newUsers.ToList()
    81      select new DimUser {
    82        UserId = u,
    83        Name = userManager.GetUserById(u) != null ? userManager.GetUserById(u).UserName : UnknownUserName
    84      };
    85 
    86       db.DimUsers.InsertAllOnSubmit(newDimUsers);
    87 
    88       // insert NULL-User
    89       if (!db.DimUsers.Any(x => x.UserId == Guid.Empty)) {
    90         db.DimUsers.InsertOnSubmit(new DimUser { UserId = Guid.Empty, Name = "NULL" });
    91       }
    92     }
    93 
    94     private void UpdateJobs(HiveDataContext db) {
    95       var newJobs =
    96         from j in db.Jobs
    97         where !db.DimJobs.Select(x => x.JobId).Contains(j.JobId)
    98         select j;
    99 
    100       var newDimJobs =
    101         from j in newJobs.ToList()
    102         select new DimJob {
    103           JobId = j.JobId,
    104           JobName = j.Name == null ? string.Empty : j.Name,
    105           UserId = j.OwnerUserId,
    106           UserName = userManager.GetUserById(j.OwnerUserId) != null ? userManager.GetUserById(j.OwnerUserId).UserName : UnknownUserName
    107         };
    108 
    109       db.DimJobs.InsertAllOnSubmit(newDimJobs);
    110 
    111       // insert NULL-Job
    112       if (!db.DimJobs.Any(x => x.JobId == Guid.Empty)) {
    113         db.DimJobs.InsertOnSubmit(new DimJob { JobId = Guid.Empty, JobName = "NULL", UserId = Guid.Empty, UserName = "NULL" });
    114       }
    115     }
    116 
    117     private void UpdateClients(HiveDataContext db) {
    118       var removedClients =
    119         from c in db.DimClients
    120         where c.ExpirationTime == null &&
    121               !db.Resources.OfType<Slave>().Select(x => x.ResourceId).Contains(c.ResourceId)
    122         select c;
    123 
    124       var modifiedClients =
    125         from s in db.Resources.OfType<Slave>()
    126         join c in db.DimClients on s.ResourceId equals c.ResourceId
    127         where c.ExpirationTime == null
    128               && (s.Name != c.Name || s.ParentResourceId != c.ResourceGroupId ||
    129                   s.ParentResource.ParentResourceId != c.ResourceGroup2Id)
    130         select new { Slave = s, Client = c };
    131 
    132       foreach (var client in removedClients.Union(modifiedClients.Select(x => x.Client))) {
    133         client.ExpirationTime = DateTime.Now;
    134       }
    135 
    136       var newClients =
    137         from s in db.Resources.OfType<Slave>()
    138         where !db.DimClients.Select(x => x.ResourceId).Contains(s.ResourceId)
    139               || modifiedClients.Select(x => x.Slave.ResourceId).Contains(s.ResourceId)
    140         select new {
    141           Slave = s,
    142           Group = s.ParentResourceId,
    143           Group2 = s.ParentResource.ParentResourceId
    144         };
    145 
    146       var newDimClients =
    147         from s in newClients.ToList()
    148         select new DimClient {
    149           ResourceId = s.Slave.ResourceId,
    150           Name = s.Slave.Name,
    151           ExpirationTime = null,
    152           ResourceGroupId = s.Group,
    153           ResourceGroup2Id = s.Group2
    154         };
    155 
    156       db.DimClients.InsertAllOnSubmit(newDimClients);
    157     }
    158 
    159     private void UpdateFactTables(DimTime newTime, HiveDataContext db) {
    160       UpdateClientInfoFacts(newTime, db);
    161       UpdateTaskFacts(newTime, db);
    162     }
    163 
    164     private void UpdateClientInfoFacts(DimTime newTime, HiveDataContext db) {
    165       var time = newTime.Time;
    166 
    167       var lastFacts =
    168         from cf in db.FactClientInfos
    169         join r in db.DimClients on cf.ClientId equals r.Id
    170         group cf by r.ResourceId into grpFacts
    171         select new {
    172           ResourceId = grpFacts.Key,
    173           Fact = grpFacts.OrderByDescending(x => x.Time).First(),
    174         };
    175 
    176       var slaves =
    177         from s in db.Resources.OfType<Slave>()
    178         join c in db.DimClients on s.ResourceId equals c.ResourceId
    179         join lcf in lastFacts on c.ResourceId equals lcf.ResourceId into joinCf
    180         from cf in joinCf.DefaultIfEmpty()
    181         where c.ExpirationTime == null
    182         select new {
    183           Slave = s,
    184           Client = c,
    185           LastFact = cf != null ? cf.Fact : null
    186         };
    187 
    188       var clientFacts =
    189         from s in slaves.ToList()
    190         let duration = s.LastFact != null ? (time - s.LastFact.Time).TotalMinutes : SmallestTimeSpan.TotalMinutes
    191         select new FactClientInfo {
    192           DimClient = s.Client,
    193           DimTime = newTime,
    194           UserId = s.Slave.OwnerUserId ?? Guid.Empty,
    195           NumUsedCores = s.Slave.Cores != null && s.Slave.FreeCores != null
    196                          ? s.Slave.Cores.Value - s.Slave.FreeCores.Value : 0,
    197           NumTotalCores = s.Slave.Cores ?? 0,
    198           UsedMemory = s.Slave.Memory != null && s.Slave.FreeMemory != null
    199                        ? s.Slave.Memory.Value - s.Slave.FreeMemory.Value : 0,
    200           TotalMemory = s.Slave.Memory ?? 0,
    201           CpuUtilization = s.Slave.CpuUtilization,
    202           SlaveState = s.Slave.SlaveState,
    203           TotalTimeIdle = s.Slave.SlaveState == SlaveState.Idle && s.Slave.IsAllowedToCalculate
    204                                   ? duration : 0.0,
    205           TotalTimeCalculating = s.Slave.SlaveState == SlaveState.Calculating
    206                                   ? duration : 0.0,
    207           TotalTimeTransferring = 0.0,
    208           TotalTimeUnavailable = s.Slave.SlaveState == SlaveState.Idle && !s.Slave.IsAllowedToCalculate
    209                                   ? duration : 0.0,
    210           TotalTimeOffline = s.Slave.SlaveState == SlaveState.Offline
    211                                   ? duration : 0.0
    212         };
    213 
    214       db.FactClientInfos.InsertAllOnSubmit(clientFacts);
    215     }
    216 
    217     private void UpdateTaskFacts(DimTime newTime, HiveDataContext db) {
    218       // old Task facts
    219       var oldFacts =
    220         from fact in db.FactTasks
    221         where fact.EndTime == null
    222         select fact;
    223 
    224       // query Task facts data
    225       var newFacts =
    226         from task in db.Tasks
    227         where !task.IsParentTask
    228           && (!db.FactTasks.Select(fact => fact.TaskId).Contains(task.TaskId) || oldFacts.Select(fact => fact.TaskId).Contains(task.TaskId))
    229         join lastFact in oldFacts on task.TaskId equals lastFact.TaskId into lastFactPerTask
     205      );
     206    }
     207
     208    private void UpdateTaskFacts(DimTime newTime, PersistenceManager pm) {
     209      var factTaskDao = pm.FactTaskDao;
     210      var taskDao = pm.TaskDao;
     211      var dimClientDao = pm.DimClientDao;
     212      var stateLogDao = pm.StateLogDao;
     213
     214      var factTaskIds = factTaskDao.GetAll().Select(x => x.TaskId);
     215      var notFinishedFactTasks = factTaskDao.GetNotFinishedTasks().Select(x => new {
     216        x.TaskId,
     217        x.DimTimeStart,
     218        x.LastClientId
     219      });
     220
     221      var newTasks =
     222        from task in taskDao.GetAllChildTasks()
     223        let stateLogs = task.StateLogs.OrderByDescending(x => x.DateTime)
     224        let lastSlaveId = stateLogs.First(x => x.SlaveId != null).SlaveId
     225        where (!factTaskIds.Contains(task.TaskId)
     226               || notFinishedFactTasks.Select(x => x.TaskId).Contains(task.TaskId))
     227        join lastFactTask in notFinishedFactTasks on task.TaskId equals lastFactTask.TaskId into lastFactPerTask
    230228        from lastFact in lastFactPerTask.DefaultIfEmpty()
    231         let lastSlaveId = task.StateLogs.OrderByDescending(sl => sl.DateTime).First(sl => sl.SlaveId != null).SlaveId
    232         join client in db.DimClients.Where(client => client.ExpirationTime == null) on lastSlaveId equals client.ResourceId into clientsPerSlaveId
     229        join client in dimClientDao.GetRecentlyAddedClients() on lastSlaveId equals client.ResourceId into clientsPerSlaveId
    233230        from client in clientsPerSlaveId.DefaultIfEmpty()
    234231        select new {
    235           Task = task,
    236           StateLogs = task.StateLogs.OrderBy(sl => sl.DateTime),
    237           LastClientId = client != null ? client.Id : default(Guid?),
    238           LastFact = lastFact
     232          TaskId = task.TaskId,
     233          JobId = task.JobId,
     234          Priority = task.Priority,
     235          CoresRequired = task.CoresNeeded,
     236          MemoryRequired = task.MemoryNeeded,
     237          State = task.State,
     238          StateLogs = stateLogs,
     239          DimTimeStart = lastFact != null ? lastFact.DimTimeStart : newTime,
     240          LastClientId = client != null
     241                         ? client.Id : lastFact != null
     242                         ? lastFact.LastClientId : (Guid?)null
    239243        };
    240244
    241       // new Task facts
    242       var newTaskFacts =
    243         from t in newFacts.ToList()
    244         let stateLogsLinkedList = new LinkedList<StateLog>(t.StateLogs)
     245      var endStates = new[] { TaskState.Finished, TaskState.Failed, TaskState.Aborted };
     246      factTaskDao.Save(
     247        from x in newTasks.ToList()
     248        let stateLogsLinkedList = new LinkedList<StateLog>(x.StateLogs.OrderBy(y => y.DateTime))
     249        let lastStateLog = stateLogsLinkedList.OrderByDescending(y => y.DateTime).FirstOrDefault(sl => sl.Exception != null)
    245250        select new FactTask {
    246           TaskId = t.Task.TaskId,
    247           JobId = t.Task.JobId,
    248           DimTimeStart = t.LastFact != null ? t.LastFact.DimTimeStart : newTime,
    249           DimTimeEnd = new[] { TaskState.Finished, TaskState.Failed, TaskState.Aborted }.Contains(t.Task.State) ? newTime : null,
    250           LastClientId = t.LastClientId ?? (t.LastFact != null ? t.LastFact.LastClientId : Guid.Empty),
    251           Priority = t.Task.Priority,
    252           CoresRequired = t.Task.CoresNeeded,
    253           MemoryRequired = t.Task.MemoryNeeded,
     251          TaskId = x.TaskId,
     252          JobId = x.JobId,
     253          DimTimeStart = x.DimTimeStart,
     254          DimTimeEnd = endStates.Contains(x.State) ? newTime : null,
     255          LastClientId = x.LastClientId,
     256          Priority = x.Priority,
     257          CoresRequired = x.CoresRequired,
     258          MemoryRequired = x.MemoryRequired,
    254259          NumCalculationRuns = stateLogsLinkedList.CountCalculationRuns(),
    255260          NumRetries = stateLogsLinkedList.CountRetries(),
    256261          TotalWaitingTime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Waiting),
    257           TotalRuntime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(x => x.State == TaskState.Transferring)),
     262          TotalRuntime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(y => y.State == TaskState.Transferring)),
    258263          TotalTransferTime = stateLogsLinkedList.SumTotalTimeWhere(stateLog => stateLog.Value.State == TaskState.Transferring),
    259           TaskState = t.Task.State,
    260           Exception = stateLogsLinkedList.FirstOrDefault(sl => sl.Exception != null) == null ? string.Empty : stateLogsLinkedList.FirstOrDefault(sl => sl.Exception != null).Exception
    261         };
    262 
    263       db.FactTasks.DeleteAllOnSubmit(oldFacts.ToList());
    264       db.FactTasks.InsertAllOnSubmit(newTaskFacts);
     264          TaskState = x.State,
     265          Exception = lastStateLog == null ? string.Empty : lastStateLog.Exception
     266        });
     267      factTaskDao.Delete(notFinishedFactTasks.Select(x => x.TaskId));
     268    }
     269
     270    private string GetUserName(Guid userId) {
     271      try {
     272        var userManager = ServiceLocator.Instance.UserManager;
     273        var user = userManager.GetUserById(userId);
     274        return user != null ? user.UserName : UnknownUserName;
     275      }
     276      catch (Exception) {
     277        return UnknownUserName;
     278      }
    265279    }
    266280  }

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Interfaces/IServiceLocator.cs

    r12441 r12468  
    2121
    2222using HeuristicLab.Services.Hive.DataAccess;
     23using HeuristicLab.Services.Hive.DataAccess.Interfaces;
    2324
    2425namespace HeuristicLab.Services.Hive {
    … …  
    2627    Access.IRoleVerifier RoleVerifier { get; }
    2728    IAuthorizationManager AuthorizationManager { get; }
     29    IPersistenceManager PersistenceManager { get; }
    2830    IHiveDao HiveDao { get; }
    2931    IOptimizedHiveDao OptimizedHiveDao { get; }

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/EventManager.cs

    r12172 r12468  
    2424using System.Linq;
    2525using HeuristicLab.Services.Hive.DataAccess;
     26using HeuristicLab.Services.Hive.DataAccess.Interfaces;
    2627using DT = HeuristicLab.Services.Hive.DataTransfer;
    2728
    … …  
    4142      get { return LogFactory.GetLogger(this.GetType().Namespace); }
    4243    }
    43     private DataAccess.ITransactionManager trans {
     44    private ITransactionManager trans {
    4445      get { return ServiceLocator.Instance.TransactionManager; }
    4546    }

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/Manager/HeartbeatManager.cs

    r12012 r12468  
    2525using System.Threading;
    2626using HeuristicLab.Services.Hive.DataAccess;
     27using HeuristicLab.Services.Hive.DataAccess.Interfaces;
    2728using Heartbeat = HeuristicLab.Services.Hive.DataTransfer.Heartbeat;
    2829
    … …  
    3738      get { return ServiceLocator.Instance.TaskScheduler; }
    3839    }
    39     private DataAccess.ITransactionManager trans {
     40    private ITransactionManager trans {
    4041      get { return ServiceLocator.Instance.TransactionManager; }
    4142    }

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/ServiceLocator.cs

    r12441 r12468  
    2121
    2222using HeuristicLab.Services.Hive.DataAccess;
     23using HeuristicLab.Services.Hive.DataAccess.Interfaces;
     24using HeuristicLab.Services.Hive.DataAccess.Manager;
    2325
    2426namespace HeuristicLab.Services.Hive {
    … …  
    3234      }
    3335      set { instance = value; }
     36    }
     37
     38    public IPersistenceManager PersistenceManager {
     39      get { return new PersistenceManager(); }
    3440    }
    3541
Note: See TracChangeset for help on using the changeset viewer.