Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
10/18/18 16:16:31 (6 years ago)
Author:
gkronber
Message:

#2915: merged changes in the trunk up to current HEAD (r15951:16232) into the branch

Location:
branches/2915-AbsoluteSymbol
Files:
1 deleted
7 edited
6 copied

Legend:

Unmodified
Added
Removed

  • branches/2915-AbsoluteSymbol

  • branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive.DataAccess

  • branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/HiveStatistics/DimClientDao.cs

    r15583 r16240  
    3333    }
    3434
    35     public IQueryable<DimClient> GetActiveClients() {
    36       return Table.Where(x => x.ExpirationTime == null);
     35    public IQueryable<DimClient> GetAllOnlineClients() {
     36      return Table.Where(x => x.DateExpired == null);
    3737    }
    3838
    39     public IQueryable<DimClient> GetExpiredClients() {
    40       return Table.Where(x => x.ExpirationTime != null);
     39    public IQueryable<DimClient> GetAllExpiredClients() {
     40      return Table.Where(x => x.DateExpired != null);
     41    }
     42
     43    public IQueryable<DimClient> GetAllOnlineSlaves() {
     44      return Table.Where(x => x.DateExpired == null && x.ResourceType == "Slave");
     45    }
     46
     47    public IQueryable<DimClient> GetAllOnlineSlaveGroups() {
     48      return Table.Where(x => x.DateExpired == null && x.ResourceType == "GROUP");
     49    }
     50
     51    public IQueryable<DimClient> GetAllExpiredSlaves() {
     52      return Table.Where(x => x.DateExpired != null && x.ResourceType == "Slave");
     53    }
     54
     55    public IQueryable<DimClient> GetAllExpiredSlaveGroups() {
     56      return Table.Where(x => x.DateExpired != null && x.ResourceType == "GROUP");
    4157    }
    4258
    … …  
    4460      string paramIds = string.Join(",", ids.Select(x => string.Format("'{0}'", x)));
    4561      if (!string.IsNullOrWhiteSpace(paramIds)) {
    46         string query = string.Format(UpdateExpirationTimeQuery, "{0}", paramIds);
     62        string query = string.Format(UpdateDateExpiredQuery, "{0}", paramIds);
    4763        return DataContext.ExecuteCommand(query, time);
    4864      }
    … …  
    5975
    6076    #region String queries
    61     private const string UpdateExpirationTimeQuery =
     77    private const string UpdateDateExpiredQuery =
    6278      @"UPDATE [statistics].[DimClient]
    63            SET ExpirationTime = {0}
     79           SET DateExpired = {0}
    6480         WHERE Id IN ({1});";
    6581    #endregion

  • branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/HiveStatistics/FactTaskDao.cs

    r15583 r16240  
    6666      return from factTask in Table
    6767             join client in DimClientTable on factTask.LastClientId equals client.Id
    68              where client.ResourceGroupId == id
     68             where client.ParentResourceId == id
    6969             select factTask;
    7070    }

  • branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/JobDao.cs

    r15583 r16240  
    2121
    2222using System;
     23using System.Collections.Generic;
    2324using System.Data.Linq;
    2425using System.Linq;
    … …  
    3233    }
    3334
     35    public void DeleteByState(JobState state) {
     36      DataContext.ExecuteCommand(DeleteByStateQueryString, Enum.GetName(typeof(JobState), state));
     37    }
     38
     39    public IEnumerable<Job> GetByProjectId(Guid id) {
     40      return GetByProjectIdQuery(DataContext, id);
     41    }
     42
     43    public IEnumerable<Job> GetByProjectIds(IEnumerable<Guid> projectIds) {
     44      string paramProjectIds = string.Join(",", projectIds.ToList().Select(x => string.Format("'{0}'", x)));
     45      if(!string.IsNullOrWhiteSpace(paramProjectIds)) {
     46        string queryString = string.Format(GetByProjectIdsQueryString, paramProjectIds);
     47        return DataContext.ExecuteQuery<Job>(queryString);
     48      }
     49      return Enumerable.Empty<Job>();
     50    }
     51
     52    public IEnumerable<Job> GetByState(JobState state) {
     53      return GetByStateQuery(DataContext, state);
     54    }
     55
     56    public IEnumerable<Guid> GetJobIdsByState(JobState state) {
     57      return GetJobIdsByStateQuery(DataContext, state);
     58    }
     59
     60    public IEnumerable<Job> GetJobsReadyForDeletion() {
     61      return GetJobsReadyForDeletionQuery(DataContext);
     62    }
     63
     64
    3465    #region Compiled queries
    3566    private static readonly Func<DataContext, Guid, Job> GetByIdQuery =
    … …  
    3869         where job.JobId == jobId
    3970         select job).SingleOrDefault());
     71    private static readonly Func<DataContext, Guid, IEnumerable<Job>> GetByProjectIdQuery =
     72      CompiledQuery.Compile((DataContext db, Guid projectId) =>
     73        (from job in db.GetTable<Job>()
     74         where job.ProjectId == projectId
     75         select job));
     76    private static readonly Func<DataContext, JobState, IEnumerable<Job>> GetByStateQuery =
     77      CompiledQuery.Compile((DataContext db, JobState jobState) =>
     78        (from job in db.GetTable<Job>()
     79         where job.State == jobState
     80         select job));
     81    private static readonly Func<DataContext, JobState, IEnumerable<Guid>> GetJobIdsByStateQuery =
     82      CompiledQuery.Compile((DataContext db, JobState jobState) =>
     83        (from job in db.GetTable<Job>()
     84         where job.State == jobState
     85         select job.JobId));
     86    private static readonly Func<DataContext, IEnumerable<Job>> GetJobsReadyForDeletionQuery =
     87      CompiledQuery.Compile((DataContext db) =>
     88        (from job in db.GetTable<Job>()
     89         where job.State == JobState.StatisticsPending
     90         && (from task in db.GetTable<Task>()
     91             where task.JobId == job.JobId
     92             select task.State == TaskState.Finished
     93              || task.State == TaskState.Aborted
     94              || task.State == TaskState.Failed).All(x => x)
     95         select job));
     96    #endregion
     97
     98    #region String queries
     99    private const string DeleteByStateQueryString = @"
     100      DELETE FROM [Job]
     101      WHERE JobState = {0}
     102    ";
     103    private const string GetStatisticsPendingJobs = @"
     104      SELECT DISTINCT j.*
     105      FROM [Job] j
     106      WHERE j.JobState = 'StatisticsPending'
     107      AND 'Calculating' NOT IN (
     108        SELECT t.TaskState
     109        FROM [Task] t
     110        WHERE t.JobId = j.JobId)
     111    ";
     112    private const string GetByProjectIdsQueryString = @"
     113      SELECT DISTINCT j.*
     114      FROM [Job] j
     115      WHERE j.ProjectId IN ({0})
     116    ";
    40117    #endregion
    41118  }

  • branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/ResourceDao.cs

    r15583 r16240  
    2121
    2222using System;
     23using System.Collections.Generic;
    2324using System.Data.Linq;
    2425using System.Linq;
    … …  
    3637    }
    3738
     39    public void DeleteByIds(IEnumerable<Guid> ids) {
     40      string paramResourceIds = string.Join(",", ids.ToList().Select(x => string.Format("'{0}'", x)));
     41      if (!string.IsNullOrWhiteSpace(paramResourceIds)) {
     42        string queryString = string.Format(DeleteByIdsQueryString, paramResourceIds);
     43        DataContext.ExecuteCommand(queryString);
     44      }
     45    }
     46
     47    public bool CheckExistence(IEnumerable<Guid> ids) {
     48      string paramResourceIds = string.Join(",", ids.ToList().Select(x => string.Format("'{0}'", x)));
     49      if (!string.IsNullOrWhiteSpace(paramResourceIds)) {
     50        string queryString = string.Format(CountExistenceQuery, paramResourceIds);
     51        return DataContext.ExecuteQuery<int>(queryString).SingleOrDefault() == ids.Count();
     52      }
     53      return false;
     54    }
     55
    3856    public IQueryable<Resource> GetResourcesWithValidOwner() {
    3957      return Table.Where(x => x.OwnerUserId != null);
     58    }
     59
     60    public IEnumerable<Resource> GetChildResourcesById(Guid id) {
     61      return DataContext.ExecuteQuery<Resource>(GetChildResourcesByIdQuery, id);
     62    }
     63
     64    public IEnumerable<Guid> GetChildResourceIdsById(Guid id) {
     65      return DataContext.ExecuteQuery<Guid>(GetChildResourceIdsByIdQuery, id);
     66    }
     67
     68    public IEnumerable<Resource> GetParentResourcesById(Guid id) {
     69      return DataContext.ExecuteQuery<Resource>(GetParentResourcesByIdQuery, id);
     70    }
     71
     72    public IEnumerable<Guid> GetParentResourceIdsById(Guid id) {
     73      return DataContext.ExecuteQuery<Guid>(GetParentResourceIdsByIdQuery, id);
     74    }
     75
     76    public IEnumerable<Resource> GetCurrentAndParentResourcesById(Guid id) {
     77      return DataContext.ExecuteQuery<Resource>(GetCurrentAndParentResourcesByIdQuery, id);
     78    }
     79
     80    public IEnumerable<Guid> GetCurrentAndParentResourceIdsById(Guid id) {
     81      return DataContext.ExecuteQuery<Guid>(GetCurrentAndParentResourceIdsByIdQuery, id);
    4082    }
    4183
    … …  
    5395        select resource).FirstOrDefault());
    5496    #endregion
     97
     98    #region String queries
     99    private const string DeleteByIdsQueryString = @"
     100      DELETE FROM [Resource]
     101      WHERE ResourceId IN ({0})
     102    ";
     103    private const string CountExistenceQuery = @"
     104      SELECT COUNT(DISTINCT r.ResourceId)
     105      FROM [Resource] r
     106      WHERE r.ResourceId IN ({0})
     107    ";
     108    private const string GetChildResourcesByIdQuery = @"
     109      WITH rtree AS
     110      (
     111        SELECT ResourceId, ParentResourceId
     112        FROM [Resource]
     113        UNION ALL
     114        SELECT rt.ResourceId, r.ParentResourceId
     115        FROM [Resource] r
     116        JOIN rtree rt ON rt.ParentResourceId = r.ResourceId AND r.ParentResourceId <> r.ResourceId AND rt.ParentResourceId <> rt.ResourceId
     117      )
     118      SELECT DISTINCT res.*
     119      FROM rtree, [Resource] res
     120      WHERE rtree.ParentResourceId = {0}
     121      AND rtree.ResourceId = res.ResourceId
     122    ";
     123    private const string GetChildResourceIdsByIdQuery = @"
     124      WITH rtree AS
     125      (
     126        SELECT ResourceId, ParentResourceId
     127        FROM [Resource]
     128        UNION ALL
     129        SELECT rt.ResourceId, r.ParentResourceId
     130        FROM [Resource] r
     131        JOIN rtree rt ON rt.ParentResourceId = r.ResourceId AND r.ParentResourceId <> r.ResourceId AND rt.ParentResourceId <> rt.ResourceId
     132      )
     133      SELECT DISTINCT rtree.ResourceId
     134      FROM rtree
     135      WHERE rtree.ParentResourceId = {0}
     136    ";
     137    private const string GetParentResourcesByIdQuery = @"
     138      WITH rbranch AS
     139      (
     140        SELECT ResourceId, ParentResourceId
     141        FROM [Resource]
     142        UNION ALL
     143        SELECT rb.ResourceId, r.ParentResourceId
     144        FROM [Resource] r
     145        JOIN rbranch rb ON rb.ParentResourceId = r.ResourceId AND r.ParentResourceId <> r.ResourceId AND rb.ParentResourceId <> rb.ResourceId
     146      )
     147      SELECT DISTINCT res.*
     148      FROM rbranch, [Resource] res
     149      WHERE rbranch.ResourceId = {0}
     150      AND rbranch.ParentResourceId = res.ResourceId
     151    ";
     152    private const string GetParentResourceIdsByIdQuery = @"
     153      WITH rbranch AS
     154      (
     155        SELECT ResourceId, ParentResourceId
     156        FROM [Resource]
     157        UNION ALL
     158        SELECT rb.ResourceId, r.ParentResourceId
     159        FROM [Resource] r
     160        JOIN rbranch rb ON rb.ParentResourceId = r.ResourceId AND r.ParentResourceId <> r.ResourceId AND rb.ParentResourceId <> rb.ResourceId
     161      )
     162      SELECT DISTINCT rbranch.ParentResourceId
     163      FROM rbranch
     164      WHERE rbranch.ResourceId = {0}
     165    ";
     166    private const string GetCurrentAndParentResourcesByIdQuery = @"
     167      WITH rbranch AS
     168      (
     169        SELECT ResourceId, ParentResourceId
     170        FROM [Resource]
     171        WHERE ResourceId = {0}
     172        UNION ALL
     173        SELECT r.ResourceId, r.ParentResourceId
     174        FROM [Resource] r
     175        JOIN rbranch rb ON rb.ParentResourceId = r.ResourceId
     176      )
     177      SELECT DISTINCT res.*
     178      FROM rbranch, [Resource] res
     179      WHERE rbranch.ResourceId = res.ResourceId
     180    ";
     181    private const string GetCurrentAndParentResourceIdsByIdQuery = @"
     182      WITH rbranch AS
     183      (
     184        SELECT ResourceId, ParentResourceId
     185        FROM [Resource]
     186        WHERE ResourceId = {0}
     187        UNION ALL
     188        SELECT r.ResourceId, r.ParentResourceId
     189        FROM [Resource] r
     190        JOIN rbranch rb ON rb.ParentResourceId = r.ResourceId
     191      )
     192      SELECT DISTINCT rbranch.ResourceId
     193      FROM rbranch
     194    ";
     195    #endregion
    55196  }
    56197}

  • branches/2915-AbsoluteSymbol/HeuristicLab.Services.Hive.DataAccess/3.3/Daos/TaskDao.cs

    r15583 r16240  
    2727namespace HeuristicLab.Services.Hive.DataAccess.Daos {
    2828  public class TaskDao : GenericDao<Guid, Task> {
    29     private Table<AssignedResource> AssignedResourceTable {
    30       get { return DataContext.GetTable<AssignedResource>(); }
    31     }
    32 
    3329    public TaskDao(DataContext dataContext) : base(dataContext) { }
    3430
    … …  
    5450      //Because there is at the moment no case where this makes sense (there don't exist parent tasks which need to be calculated),
    5551      //we skip this step because it's wasted runtime
    56       return DataContext.ExecuteQuery<TaskPriorityInfo>(GetWaitingTasksQueryString, slave.ResourceId, Enum.GetName(typeof(TaskState), TaskState.Waiting), slave.FreeCores, slave.FreeMemory).ToList();
     52      return DataContext.ExecuteQuery<TaskPriorityInfo>(GetWaitingTasksQueryString,
     53        slave.ResourceId,
     54        Enum.GetName(typeof(TaskState), TaskState.Waiting),
     55        slave.FreeCores,
     56        slave.FreeMemory).ToList();
    5757    }
    5858
    … …  
    6262    /// <param name="resourceIds">list of resourceids which for which the task should be valid</param>
    6363    /// <param name="count">maximum number of task to return</param>
    64     /// <param name="finished">if true, all parent task which have FinishWhenChildJobsFinished=true are returned, otherwise only FinishWhenChildJobsFinished=false are returned</param>
     64    /// <param name="finished">if true, all parent tasks which have FinishWhenChildJobsFinished=true are returned, otherwise only FinishWhenChildJobsFinished=false are returned</param>
    6565    /// <returns></returns>
    6666    public IEnumerable<Task> GetParentTasks(IEnumerable<Guid> resourceIds, int count, bool finished) {
    67       var query = from ar in AssignedResourceTable
    68                   where resourceIds.Contains(ar.ResourceId)
    69                      && ar.Task.State == TaskState.Waiting
    70                      && ar.Task.IsParentTask
    71                      && (finished ? ar.Task.FinishWhenChildJobsFinished : !ar.Task.FinishWhenChildJobsFinished)
    72                      && (from child in Table
    73                          where child.ParentTaskId == ar.Task.TaskId
    74                          select child.State == TaskState.Finished
    75                              || child.State == TaskState.Aborted
    76                              || child.State == TaskState.Failed).All(x => x)
    77                      && (from child in Table // avoid returning WaitForChildTasks task where no child-task exist (yet)
    78                          where child.ParentTaskId == ar.Task.TaskId
    79                          select child).Any()
    80                   orderby ar.Task.Priority descending
    81                   select ar.Task;
     67    var query = from t in Table
     68                where t.State == TaskState.Waiting
     69                    && t.IsParentTask
     70                    && t.Job.AssignedJobResources.All(x => resourceIds.ToList().Contains(x.ResourceId))
     71                    && t.FinishWhenChildJobsFinished == finished
     72                    && t.ChildJobs.Any()
     73                    && t.ChildJobs.All(x =>
     74                      x.State == TaskState.Finished
     75                      || x.State == TaskState.Aborted
     76                      || x.State == TaskState.Failed)
     77                  orderby t.Priority descending
     78                  select t;
    8279      return count == 0 ? query.ToArray() : query.Take(count).ToArray();
    8380    }
    … …  
    9693
    9794    #region String queries
     95    private const string GetCalculatingChildTasksByProjectId = @"
     96      SELECT t.* FROM [Task] t, [Job] j
     97        WHERE t.IsParentTask = 0
     98        AND t.TaskState = 'Calculating'
     99        AND t.JobId = j.JobId
     100        AND j.ProjectId = {0}
     101        ORDER BY j.ProjectId
     102    ";
    98103    private const string GetWaitingTasksQueryString = @"
    99       WITH pr AS (
     104      WITH rbranch AS (
    100105        SELECT ResourceId, ParentResourceId
    101106        FROM [Resource]
    … …  
    103108        UNION ALL
    104109        SELECT r.ResourceId, r.ParentResourceId
    105         FROM [Resource] r JOIN pr ON r.ResourceId = pr.ParentResourceId
     110        FROM [Resource] r
     111        JOIN rbranch rb ON rb.ParentResourceId = r.ResourceId
    106112      )
    107113      SELECT DISTINCT t.TaskId, t.JobId, t.Priority
    108       FROM pr JOIN AssignedResources ar ON ar.ResourceId = pr.ResourceId
    109           JOIN Task t ON t.TaskId = ar.TaskId
     114      FROM [Task] t, [Job] j, [AssignedJobResource] ajr, rbranch
    110115      WHERE NOT (t.IsParentTask = 1 AND t.FinishWhenChildJobsFinished = 1)
    111           AND t.TaskState = {1}
    112           AND t.CoresNeeded <= {2}
    113           AND t.MemoryNeeded <= {3}
     116      AND t.TaskState = {1}
     117      AND t.CoresNeeded <= {2}
     118      AND t.MemoryNeeded <= {3}
     119      AND t.JobId = j.JobId
     120      AND j.JobState = 'Online'
     121      AND j.JobId = ajr.JobId
     122      AND ajr.ResourceId = rbranch.ResourceId
    114123    ";
    115124
Note: See TracChangeset for help on using the changeset viewer.