Free cookie consent management tool by TermsFeed Policy Generator

Changeset 9571


Ignore:
Timestamp:
06/03/13 15:18:12 (11 years ago)
Author:
pfleck
Message:

#2063:
Calculating total waiting-, transfer- and calculating time for task statistics.

Location:
branches/HiveStatistics/sources
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.dbml

    r9557 r9571  
    230230      <Column Name="TotalTransferTime" Type="System.Double" DbType="Float NOT NULL" CanBeNull="false" />
    231231      <Column Name="NumCalculationRuns" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
    232       <Column Name="NumFails" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
     232      <Column Name="NumRetries" Storage="_NumFails" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
    233233      <Column Name="CoresRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
    234234      <Column Name="MemoryRequired" Type="System.Int32" DbType="Int NOT NULL" CanBeNull="false" />
  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/HiveDataContext.designer.cs

    r9557 r9571  
    49084908    partial void OnNumCalculationRunsChanging(int value);
    49094909    partial void OnNumCalculationRunsChanged();
    4910     partial void OnNumFailsChanging(int value);
    4911     partial void OnNumFailsChanged();
     4910    partial void OnNumRetriesChanging(int value);
     4911    partial void OnNumRetriesChanged();
    49124912    partial void OnCoresRequiredChanging(int value);
    49134913    partial void OnCoresRequiredChanged();
     
    50405040   
    50415041    [global::System.Data.Linq.Mapping.ColumnAttribute(Storage="_NumFails", DbType="Int NOT NULL")]
    5042     public int NumFails
     5042    public int NumRetries
    50435043    {
    50445044      get
     
    50505050        if ((this._NumFails != value))
    50515051        {
    5052           this.OnNumFailsChanging(value);
     5052          this.OnNumRetriesChanging(value);
    50535053          this.SendPropertyChanging();
    50545054          this._NumFails = value;
    5055           this.SendPropertyChanged("NumFails");
    5056           this.OnNumFailsChanged();
     5055          this.SendPropertyChanged("NumRetries");
     5056          this.OnNumRetriesChanged();
    50575057        }
    50585058      }
  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive.DataAccess/3.3/SQL Scripts/Initialize Hive Database.sql

    r9557 r9571  
    258258    [TotalTransferTime]  FLOAT (53)       NOT NULL,
    259259    [NumCalculationRuns] INT              NOT NULL,
    260     [NumFails]           INT              NOT NULL,
     260    [NumRetries]         INT              NOT NULL,
    261261    [CoresRequired]      INT              NOT NULL,
    262262    [MemoryRequired]     INT              NOT NULL,
  • branches/HiveStatistics/sources/HeuristicLab.Services.Hive/3.3/HiveStatisticsGenerator.cs

    r9561 r9571  
    11using System;
     2using System.Collections.Generic;
    23using System.Linq;
    34using System.Transactions;
     
    128129        from s in db.Resources.OfType<Slave>()
    129130        where !db.DimClients.Select(x => x.ResourceId).Contains(s.ResourceId)
    130           || modifiedClients.Select(x => x.Slave.ResourceId).Contains(s.ResourceId)
     131              || modifiedClients.Select(x => x.Slave.ResourceId).Contains(s.ResourceId)
    131132        select new {
    132133          Slave = s,
     
    222223
    223224    private void UpdateTaskFacts(DimTime newTime, HiveDataContext db) {
     225      var time = newTime.Time;
     226
    224227      var lastFacts =
    225228        from fact in db.FactTasks
     
    231234        join lastFact in lastFacts on task.TaskId equals lastFact.TaskId into lastFactsPerTask
    232235        from lastFact in lastFactsPerTask.DefaultIfEmpty()
    233         let newStateLogs = task.StateLogs.Where(stateLog => (lastFact == null || stateLog.DateTime > lastFact.EndTime) && stateLog.DateTime < newTime.Time)
     236        let newStateLogs = task.StateLogs.Where(stateLog => (lastFact == null || stateLog.DateTime > lastFact.EndTime) && stateLog.DateTime < time)
     237        let lastStateLog = lastFact == null ? null : (task.StateLogs.Where(stateLog => stateLog.DateTime < lastFact.EndTime).OrderByDescending(stateLog => stateLog.DateTime).First())
    234238        let lastSlaveId = newStateLogs.OrderByDescending(stateLog => stateLog.DateTime).First(stateLog => stateLog.SlaveId != null).SlaveId
    235239        join client in db.DimClients.Where(client => client.ExpirationTime == null) on lastSlaveId equals client.ResourceId into clientPerTask
    236240        from client in clientPerTask.DefaultIfEmpty()
     241        where newStateLogs.Any()
    237242        select new {
    238243          Task = task,
     
    240245          LastFact = lastFact,
    241246          StateLogs = newStateLogs.OrderBy(stateLog => stateLog.DateTime),
     247          LastStateLog = lastStateLog,
    242248          LastClientId = client != null ? client.Id : default(Guid?)
    243249        };
     
    245251      var newTaskFacts =
    246252        from t in lastFactAndStateLogsAndLastClientPerTask.ToList()
     253        let stateLogsLinkedList = t.LastStateLog == null ? new LinkedList<StateLog>(t.StateLogs) : new LinkedList<StateLog>(t.StateLogs).AddFirst(t.LastStateLog).List
    247254        select new FactTask {
    248255          TaskId = t.Task.TaskId,
     
    256263          TaskSize = t.LastFact != null ? t.LastFact.TaskSize : t.TaskDataSize,
    257264          ResultSize = t.Task.State == TaskState.Finished ? t.TaskDataSize : default(long?),
    258           NumCalculationRuns = (t.LastFact != null ? t.LastFact.NumCalculationRuns : 0) + t.StateLogs.Count(x => x.State == TaskState.Calculating),
    259           NumFails = (t.LastFact != null ? t.LastFact.NumFails : 0) + t.StateLogs.Count(x => x.State == TaskState.Failed || !string.IsNullOrEmpty(x.Exception)),
    260           TotalWaitingTime = 0, //ToDo
    261           TotalRuntime = 0, //ToDo
    262           TotalTransferTime = 0 //ToDo
     265          NumCalculationRuns =
     266            (t.LastFact != null ? t.LastFact.NumCalculationRuns : 0) +
     267            t.StateLogs.Count(x => x.State == TaskState.Calculating),
     268          NumRetries =
     269            (t.LastFact != null ? t.LastFact.NumRetries : 0) +
     270            stateLogsLinkedList.EnumerateNodes().Count(sl => sl.Value.State == TaskState.Calculating && sl.Next != null && !sl.NextIs(nsl => nsl.State == TaskState.Transferring)),       //TotalWaitingTime = SumTotalWaitingTime(t.LastFact, stateLogsLinkedList),
     271          TotalWaitingTime = SumTotalTime(t.LastFact, fact => fact.TotalWaitingTime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Waiting),
     272          TotalRuntime = SumTotalTime(t.LastFact, fact => fact.TotalRuntime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Calculating && stateLog.NextIs(x => x.State == TaskState.Transferring)),
     273          TotalTransferTime = SumTotalTime(t.LastFact, fact => fact.TotalTransferTime, stateLogsLinkedList, stateLog => stateLog.Value.State == TaskState.Transferring)
    263274        };
    264275
    265276      db.FactTasks.InsertAllOnSubmit(newTaskFacts);
     277    }
     278
     279    private double SumTotalTime(FactTask lastFact, Func<FactTask, double> lastValue, LinkedList<StateLog> stateLogs, Predicate<LinkedListNode<StateLog>> predicate) {
     280      double sum = lastFact != null ? lastValue(lastFact) : 0.0;
     281
     282      sum += stateLogs.EnumerateNodes()
     283                      .Where(sl => predicate(sl))
     284                      .Sum(stateLog => stateLog.Next != null ? (stateLog.Next.Value.DateTime - stateLog.Value.DateTime).TotalMinutes : 0);
     285      return sum;
     286    }
     287  }
     288
     289  public static class LinkedListNodeExtensions {
     290    public static IEnumerable<LinkedListNode<T>> EnumerateNodes<T>(this LinkedList<T> list) {
     291      var node = list.First;
     292      while (node != null) {
     293        yield return node;
     294        node = node.Next;
     295      }
     296    }
     297
     298    public static bool NextIs<T>(this LinkedListNode<T> node, Predicate<T> predicate) {
     299      return node.Next != null && predicate(node.Next.Value);
    266300    }
    267301  }
Note: See TracChangeset for help on using the changeset viewer.