Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
06/04/13 13:18:10 (11 years ago)
Author:
pfleck
Message:

#2063:
Task statistics contains only current state and no change history.
Statistics calculation is no longer based on increment updates.
TaskId is primary key in task statistics and EndTime is nullable.
Added TaskState to Task statistics.

Location:
branches/HiveStatistics/sources
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml

    r9571 r9578  
    236236      <Column Name="ResultSize" Type="System.Int64" DbType="BigInt" CanBeNull="true" />
    237237      <Column Name="Priority" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
    238       <Column Name="LastClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" CanBeNull="false" />
    239       <Column Name="JobId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" IsPrimaryKey="true" CanBeNull="false" />
    240       <Column Name="StartTime" Type="System.DateTime" DbType="DateTime NOT NULL" IsPrimaryKey="true" CanBeNull="false" />
    241       <Column Name="EndTime" Type="System.DateTime" DbType="DateTime NOT NULL" IsPrimaryKey="true" CanBeNull="false" />
     238      <Column Name="LastClientId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="false" />
     239      <Column Name="JobId" Type="System.Guid" DbType="UniqueIdentifier NOT NULL" CanBeNull="false" />
     240      <Column Name="StartTime" Type="System.DateTime" DbType="DateTime NOT NULL" CanBeNull="false" />
     241      <Column Name="EndTime" Type="System.DateTime" DbType="DateTime NOT NULL" CanBeNull="true" />
     242      <Column Name="TaskState" Type="global::HeuristicLab.Services.Hive.DataAccess.TaskState" DbType="varchar(30)" CanBeNull="false" />
    242243      <Association Name="DimClient_FactTask" Member="DimClient" ThisKey="LastClientId" OtherKey="Id" Type="DimClient" IsForeignKey="true" />
    243244      <Association Name="DimJob_FactTask" Member="DimJob" ThisKey="JobId" OtherKey="JobId" Type="DimJob" IsForeignKey="true" />

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml.layout

    r9557 r9578  
    253253      </nestedChildShapes>
    254254    </classShape>
    255     <classShape Id="04f52807-ce17-4d65-bd23-cc38c6dfbd7a" absoluteBounds="6.875, 11.25, 2, 3.2326684570312523">
     255    <classShape Id="04f52807-ce17-4d65-bd23-cc38c6dfbd7a" absoluteBounds="6.875, 11.25, 2, 3.3970157877604166">
    256256      <DataClassMoniker Name="/HiveDataContext/FactTask" />
    257257      <nestedChildShapes>
    258         <elementListCompartment Id="63e3ddcb-a6fe-48e0-a674-e650a55a9f2c" absoluteBounds="6.8900000000000006, 11.71, 1.9700000000000002, 2.67266845703125" name="DataPropertiesCompartment" titleTextColor="Black" itemTextColor="Black" />
     258        <elementListCompartment Id="63e3ddcb-a6fe-48e0-a674-e650a55a9f2c" absoluteBounds="6.8900000000000006, 11.71, 1.9700000000000002, 2.8370157877604165" name="DataPropertiesCompartment" titleTextColor="Black" itemTextColor="Black" />
    259259      </nestedChildShapes>
    260260    </classShape>
    … …  
    283283      </nestedChildShapes>
    284284    </classShape>
    285     <associationConnector edgePoints="[(6.625 : 16.1191239261004); (7.875 : 16.1191239261004); (7.875 : 14.4826684570313)]" fixedFrom="NotFixed" fixedTo="NotFixed">
     285    <associationConnector edgePoints="[(6.625 : 15.25); (6.875 : 14.6470157877604)]" fixedFrom="NotFixed" fixedTo="NotFixed">
    286286      <AssociationMoniker Name="/HiveDataContext/DimClient/DimClient_FactTask" />
    287287      <nodes>
    … …  
    290290      </nodes>
    291291    </associationConnector>
    292     <associationConnector edgePoints="[(8.5 : 10.5498478190104); (8.5 : 11.25)]" fixedFrom="NotFixed" fixedTo="NotFixed">
     292    <associationConnector edgePoints="[(8.5 : 10.5498478190104); (8.5 : 11.25)]" fixedFrom="Algorithm" fixedTo="Algorithm">
    293293      <AssociationMoniker Name="/HiveDataContext/DimJob/DimJob_FactTask" />
    294294      <nodes>
    … …  
    297297      </nodes>
    298298    </associationConnector>
    299     <associationConnector edgePoints="[(5.15625 : 10.0891951497396); (5.15625 : 14.3889184570313); (6.875 : 14.3889184570313)]" fixedFrom="NotFixed" fixedTo="NotFixed">
     299    <associationConnector edgePoints="[(5.02343625 : 10.0891951497396); (5.02343625 : 14.4398396223958); (6.875 : 14.4398396223958)]" fixedFrom="Algorithm" fixedTo="Algorithm">
    300300      <AssociationMoniker Name="/HiveDataContext/DimTime/DimTime_FactTask" />
    301301      <nodes>
    … …  
    304304      </nodes>
    305305    </associationConnector>
    306     <associationConnector edgePoints="[(5.4218775 : 10.0891951497396); (5.4218775 : 13.3576659570313); (6.71675495975 : 13.3576659570313); (6.71675495975 : 13.359301744963); (6.875 : 13.359301744963)]" manuallyRouted="true" fixedFrom="NotFixed" fixedTo="NotFixed">
     306    <associationConnector edgePoints="[(5.4218775 : 10.0891951497396); (5.4218775 : 13.3576659570313); (6.71675495975 : 13.3576659570313); (6.71675495975 : 13.4665376449926); (6.875 : 13.4665376449926)]" manuallyRouted="true" fixedFrom="NotFixed" fixedTo="NotFixed">
    307307      <AssociationMoniker Name="/HiveDataContext/DimTime/DimTime_FactTask1" />
    308308      <nodes>

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs

    r9571 r9578  
    48844884    private System.DateTime _StartTime;
    48854885   
    4886     private System.DateTime _EndTime;
     4886    private System.Nullable<System.DateTime> _EndTime;
     4887   
     4888    private global::HeuristicLab.Services.Hive.DataAccess.TaskState _TaskState;
    48874889   
    48884890    private EntityRef<DimClient> _DimClient;
    … …  
    49264928    partial void OnStartTimeChanging(System.DateTime value);
    49274929    partial void OnStartTimeChanged();
    4928     partial void OnEndTimeChanging(System.DateTime value);
     4930    partial void OnEndTimeChanging(System.Nullable<System.DateTime> value);
    49294931    partial void OnEndTimeChanged();
     4932    partial void OnTaskStateChanging(global::HeuristicLab.Services.Hive.DataAccess.TaskState value);
     4933    partial void OnTaskStateChanged();
    49304934    #endregion
    49314935   
    … …  
    51595163    }
    51605164   
    5161     [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_LastClientId", DbType="UniqueIdentifier NOT NULL", IsPrimaryKey=true)]
     5165    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_LastClientId", DbType="UniqueIdentifier NOT NULL")]
    51625166    public System.Guid LastClientId
    51635167    {
    … …  
    51835187    }
    51845188   
    5185     [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_JobId", DbType="UniqueIdentifier NOT NULL", IsPrimaryKey=true)]
     5189    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_JobId", DbType="UniqueIdentifier NOT NULL")]
    51865190    public System.Guid JobId
    51875191    {
    … …  
    52075211    }
    52085212   
    5209     [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_StartTime", DbType="DateTime NOT NULL", IsPrimaryKey=true)]
     5213    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_StartTime", DbType="DateTime NOT NULL")]
    52105214    public System.DateTime StartTime
    52115215    {
    … …  
    52315235    }
    52325236   
    5233     [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_EndTime", DbType="DateTime NOT NULL", IsPrimaryKey=true)]
    5234     public System.DateTime EndTime
     5237    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_EndTime", DbType="DateTime NOT NULL")]
     5238    public System.Nullable<System.DateTime> EndTime
    52355239    {
    52365240      get
    … …  
    52555259    }
    52565260   
     5261    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_TaskState", DbType="varchar(30)", CanBeNull=false)]
     5262    public global::HeuristicLab.Services.Hive.DataAccess.TaskState TaskState
     5263    {
     5264      get
     5265      {
     5266        return this._TaskState;
     5267      }
     5268      set
     5269      {
     5270        if ((this._TaskState != value))
     5271        {
     5272          this.OnTaskStateChanging(value);
     5273          this.SendPropertyChanging();
     5274          this._TaskState = value;
     5275          this.SendPropertyChanged("TaskState");
     5276          this.OnTaskStateChanged();
     5277        }
     5278      }
     5279    }
     5280   
    52575281    [global::System.Data.Linq.Mapping.AssociationAttribute(Name="DimClient_FactTask", Storage="_DimClient", ThisKey="LastClientId", OtherKey="Id", IsForeignKey=true)]
    52585282    public DimClient DimClient
    … …  
    53505374          else
    53515375          {
    5352             this._EndTime = default(System.DateTime);
     5376            this._EndTime = default(Nullable<System.DateTime>);
    53535377          }
    53545378          this.SendPropertyChanged("DimTimeEnd");

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql

    r9571 r9578  
    267267    [JobId]              UNIQUEIDENTIFIER NOT NULL,
    268268    [StartTime]          DATETIME         NOT NULL,
    269     [EndTime]            DATETIME         NOT NULL,
    270     CONSTRAINT [PK_FactTask] PRIMARY KEY CLUSTERED ([TaskId] ASC, [LastClientId] ASC, [JobId] ASC, [StartTime] ASC, [EndTime] ASC),
     269    [EndTime]            DATETIME         NULL,
     270  [TaskState]          VARCHAR (30)     NOT NULL,
     271    CONSTRAINT [PK_FactTask] PRIMARY KEY CLUSTERED ([TaskId] ASC),
    271272    CONSTRAINT [FK_FactTask_DimClient] FOREIGN KEY ([LastClientId]) REFERENCES [statistics].[DimClient] ([Id]),
    272273    CONSTRAINT [FK_FactTask_DimJob] FOREIGN KEY ([JobId]) REFERENCES [statistics].[DimJob] ([JobId]),

  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

    r9571 r9578  
    223223
    224224    private void UpdateTaskFacts(DimTime newTime, HiveDataContext db) {
    225       var time = newTime.Time;
    226 
    227       var lastFacts =
     225      // old Task facts
     226      var oldFacts =
    228227        from fact in db.FactTasks
    229         group fact by fact.TaskId into factsPerTask
    230         select factsPerTask.OrderByDescending(x => x.EndTime).First();
    231 
    232       var lastFactAndStateLogsAndLastClientPerTask =
    233         from task in db.Tasks.Where(task => !task.IsParentTask)
    234         join lastFact in lastFacts on task.TaskId equals lastFact.TaskId into lastFactsPerTask
    235         from lastFact in lastFactsPerTask.DefaultIfEmpty()
    236         let newStateLogs = task.StateLogs.Where(stateLog => (lastFact == null || stateLog.DateTime > lastFact.EndTime) && stateLog.DateTime < time)
    237         let lastStateLog = lastFact == null ? null : (task.StateLogs.Where(stateLog => stateLog.DateTime < lastFact.EndTime).OrderByDescending(stateLog => stateLog.DateTime).First())
    238         let lastSlaveId = newStateLogs.OrderByDescending(stateLog => stateLog.DateTime).First(stateLog => stateLog.SlaveId != null).SlaveId
    239         join client in db.DimClients.Where(client => client.ExpirationTime == null) on lastSlaveId equals client.ResourceId into clientPerTask
    240         from client in clientPerTask.DefaultIfEmpty()
    241         where newStateLogs.Any()
     228        where fact.EndTime == null
     229        select fact;
     230
     231      // query Task facts data
     232      var newFacts =
     233        from task in db.Tasks
     234        where !task.IsParentTask
     235          && (!db.FactTasks.Select(fact => fact.TaskId).Contains(task.TaskId) || oldFacts.Select(fact => fact.TaskId).Contains(task.TaskId))
     236        join lastFact in oldFacts on task.TaskId equals lastFact.TaskId into lastFactPerTask
     237        from lastFact in lastFactPerTask.DefaultIfEmpty()
     238        let lastSlaveId = task.StateLogs.OrderByDescending(sl => sl.DateTime).First(sl => sl.SlaveId != null).SlaveId
     239        join client in db.DimClients.Where(client => client.ExpirationTime == null) on lastSlaveId equals client.ResourceId into clientsPerSlaveId
     240        from client in clientsPerSlaveId.DefaultIfEmpty()
    242241        select new {
    243242          Task = task,
    244243          TaskDataSize = task.JobData.DataSize,
    245           LastFact = lastFact,
    246           StateLogs = newStateLogs.OrderBy(stateLog => stateLog.DateTime),
    247           LastStateLog = lastStateLog,
    248           LastClientId = client != null ? client.Id : default(Guid?)
    249         };
    250 
     244          StateLogs = task.StateLogs.OrderBy(sl => sl.DateTime),
     245          LastClientId = client != null ? client.Id : default(Guid?),
     246          LastFact = lastFact
     247        };
     248
     249      // new Task facts
    251250      var newTaskFacts =
    252         from t in lastFactAndStateLogsAndLastClientPerTask.ToList()
    253         let stateLogsLinkedList = t.LastStateLog == null ? new LinkedList<StateLog>(t.StateLogs) : new LinkedList<StateLog>(t.StateLogs).AddFirst(t.LastStateLog).List
     251        from t in newFacts.ToList()
     252        let stateLogsLinkedList = new LinkedList<StateLog>(t.StateLogs)
    254253        select new FactTask {
    255254          TaskId = t.Task.TaskId,
    256255          JobId = t.Task.JobId,
    257256          DimTimeStart = t.LastFact != null ? t.LastFact.DimTimeStart : newTime,
    258           DimTimeEnd = newTime,
     257          DimTimeEnd = new[] { TaskState.Finished, TaskState.Failed, TaskState.Aborted }.Contains(t.Task.State) ? newTime : null,
    259258          LastClientId = t.LastClientId ?? (t.LastFact != null ? t.LastFact.LastClientId : Guid.Empty),
    260259          Priority = t.Task.Priority,
    261260          CoresRequired = t.Task.CoresNeeded,
    262261          MemoryRequired = t.Task.MemoryNeeded,
    263           TaskSize = t.LastFact != null ? t.LastFact.TaskSize : t.TaskDataSize,
     262          TaskSize = t.LastFact == null ? t.TaskDataSize : t.LastFact.TaskSize,
    264263          ResultSize = t.Task.State == TaskState.Finished ? t.TaskDataSize : default(long?),
    265           NumCalculationRuns =
    266             (t.LastFact != null ? t.LastFact.NumCalculationRuns : 0) +
    267             t.StateLogs.Count(x => x.State == TaskState.Calculating),
    268           NumRetries =
    269             (t.LastFact != null ? t.LastFact.NumRetries : 0) +
    270             stateLogsLinkedList.EnumerateNodes().Count(sl => sl.Value.State == TaskState.Calculating && sl.Next != null && !sl.NextIs(nsl => nsl.State == TaskState.Transferring)),       //TotalWaitingTime = SumTotalWaitingTime(t.LastFact, stateLogsLinkedList),
    271           TotalWaitingTime = SumTotalTime(t.LastFact, fact => fact.TotalWaitingTime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Waiting),
    272           TotalRuntime = SumTotalTime(t.LastFact, fact => fact.TotalRuntime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(x => x.State == TaskState.Transferring)),
    273           TotalTransferTime = SumTotalTime(t.LastFact, fact => fact.TotalTransferTime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Transferring)
    274         };
    275 
     264          NumCalculationRuns = stateLogsLinkedList.EnumerateNodes()
     265            .Count(sl => sl.Value.State == TaskState.Calculating && sl.NextIs(nsl => nsl.State == TaskState.Transferring)),
     266          NumRetries = stateLogsLinkedList.EnumerateNodes()
     267            .Count(sl => sl.Value.State == TaskState.Calculating && sl.Next != null && sl.NextIs(nsl => nsl.State != TaskState.Transferring)),
     268          TotalWaitingTime = SumTotalTime(stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Waiting),
     269          TotalRuntime = SumTotalTime(stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(x => x.State == TaskState.Transferring)),
     270          TotalTransferTime = SumTotalTime(stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Transferring),
     271          TaskState = t.Task.State
     272        };
     273
     274      db.FactTasks.DeleteAllOnSubmit(oldFacts.ToList());
    276275      db.FactTasks.InsertAllOnSubmit(newTaskFacts);
    277276    }
    278277
    279     private double SumTotalTime(FactTask lastFact, Func<FactTask, double> lastValue, LinkedList<StateLog> stateLogs, Predicate<LinkedListNode<StateLog>> predicate) {
    280       double sum = lastFact != null ? lastValue(lastFact) : 0.0;
    281 
    282       sum += stateLogs.EnumerateNodes()
     278    private double SumTotalTime(LinkedList<StateLog> stateLogs, Predicate<LinkedListNode<StateLog>> predicate) {
     279      return stateLogs.EnumerateNodes()
    283280                      .Where(sl => predicate(sl))
    284281                      .Sum(stateLog => stateLog.Next != null ? (stateLog.Next.Value.DateTime - stateLog.Value.DateTime).TotalMinutes : 0);
    285       return sum;
    286282    }
    287283  }
Note: See TracChangeset for help on using the changeset viewer.