source: branches/HiveStatistics/sources/HeuristicLab.Services.Hive.Statistics/3.3/Controllers/ChartDataController.cs @ 11552

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2013 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections.Generic;
using System.Linq;
using System.Web.Mvc;
using System.Web.Script.Serialization;
using HeuristicLab.Services.Hive.DataAccess;

namespace HeuristicLab.Services.Hive.Statistics.Controllers {
  //[OutputCache(Duration = 60 * 5)]
  public class ChartDataController : Controller {
    private static readonly TimeSpan DefaultDuration = new TimeSpan(1, 0, 0, 0);

    public JsonResult AverageCpuUtilization(DateTime? start = null, DateTime? end = null) {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString)) {
        //If no given start date, get date of most recent record
        if (start == null) {
          start = GetRecentDate(db);
        }
        var data =
          from facts in GetClientFacts(db, start, end)
          select new {
            Time = facts.Key,
            CpuUtilization = facts.Average(x => x.CpuUtilization)
          };

        return Json(
          CreateSeriesData(data.ToList(), x => x.Time, x => x.CpuUtilization),
          JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult UsedCores(DateTime? start = null, DateTime? end = null) {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString)) {
        //If no given start date, get date of most recent record
        if (start == null) {
          start = GetRecentDate(db);
        }
        var data =
          from facts in GetClientFacts(db, start, end)
          select new {
            Time = facts.Key,
            UsedCores = facts.Sum(x => x.NumUsedCores),
            TotalCores = facts.Sum(x => x.NumTotalCores)
          };

        return Json(
          CreateSeriesData(data.ToList(), x => x.Time, x => x.UsedCores, x => x.TotalCores),
          JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult UsedMemory(DateTime? start = null, DateTime? end = null) {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString)) {
        //If no given start date, get date of most recent record
        if (start == null) {
          start = GetRecentDate(db);
        }
        var data =
          from facts in GetClientFacts(db, start, end)
          select new {
            Time = facts.Key,
            UsedMemory = facts.Sum(x => x.UsedMemory),
            TotalMemory = facts.Sum(x => x.TotalMemory)
          };

        // Return values in GB
        return Json(
          CreateSeriesData(data.ToList(), x => x.Time, x => x.UsedMemory / 1000, x => x.TotalMemory / 1000),
          JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult CurrentCpuUtilization(string state)
    {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
      {
        var currentCpuUtil = (from client in db.FactClientInfos
                             join clients in db.DimClients
                               on client.ClientId equals clients.Id
                             join resource in db.Resources.OfType<Slave>()
                               on clients.ResourceId equals resource.ResourceId
                             where clients.ExpirationTime == null &&
                             (state == "All") ||
                             (state == "Online") &&
                             (client.SlaveState == SlaveState.Idle || client.SlaveState == SlaveState.Calculating ) ||
                             (state == "Available") &&
                             (client.SlaveState == SlaveState.Idle)
                             select new { client.ClientId, client.CpuUtilization, client.Time })
                             .OrderBy(c => c.Time).GroupBy(c => c.ClientId).ToList();

        List<double> result = new List<double>();
        result.Add(Math.Round(currentCpuUtil.Average(s => s.Last().CpuUtilization), 2));

        return Json(result, JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult CurrentCores(string state)
    {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
      {
        var currentCores = (from client in db.FactClientInfos
                            join clients in db.DimClients
                              on client.ClientId equals clients.Id
                            join resource in db.Resources.OfType<Slave>()
                             on clients.ResourceId equals resource.ResourceId
                            where clients.ExpirationTime == null &&
                            (state == "All") ||
                            (state == "Online") &&
                            (client.SlaveState == SlaveState.Idle || client.SlaveState == SlaveState.Calculating) ||
                            (state == "Available") &&
                            (client.SlaveState == SlaveState.Idle)
                            select new { client.ClientId, client.NumTotalCores, client.NumUsedCores, client.Time })
                            .OrderBy(c => c.Time).GroupBy(c => c.ClientId).ToList();

        List<int> result = new List<int>();
        result.Add(currentCores.Sum(c => c.Last().NumTotalCores));
        result.Add(currentCores.Sum(s => s.Last().NumUsedCores));

        return Json(result, JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult CurrentMemory(string state)
    {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
      {
        var currentMemory = (from client in db.FactClientInfos
                            join clients in db.DimClients
                              on client.ClientId equals clients.Id
                            join resource in db.Resources.OfType<Slave>()
                              on clients.ResourceId equals resource.ResourceId
                            where clients.ExpirationTime == null &&
                            (state == "All") ||
                            (state == "Online") &&
                            (client.SlaveState == SlaveState.Idle || client.SlaveState == SlaveState.Calculating) ||
                            (state == "Available") &&
                            (client.SlaveState == SlaveState.Idle)
                            select new { client.ClientId, client.TotalMemory, client.UsedMemory, client.Time })
                            .OrderBy(c => c.Time).GroupBy(c => c.ClientId).ToList();

        List<int> result = new List<int>();
        result.Add(currentMemory.Sum(c => c.Last().TotalMemory)/1000);
        result.Add(currentMemory.Sum(s => s.Last().UsedMemory)/1000);

        return Json(result, JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult GetTasks(string userName, DateTime? start = null, DateTime? end = null, string jobId = null, string taskState = null)
    {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
      {
        TaskState? state = GetTaskState(taskState);

        var data =
          (from tasks in db.FactTasks
           join jobs in db.DimJobs
             on tasks.JobId equals jobs.JobId
           where jobs.UserName == userName &&
           (!start.HasValue || tasks.StartTime >= start) &&
           (!end.HasValue || tasks.EndTime < end) &&
           (string.IsNullOrEmpty(jobId) || tasks.JobId.ToString() == jobId) &&
           (string.IsNullOrEmpty(taskState) || tasks.TaskState == state)
           select new
           {
             JobName = jobs.JobName,
             JobId = jobs.JobId,
             TaskId = tasks.TaskId,
             StartDate = tasks.StartTime
           }).OrderByDescending(s => s.StartDate).ToList();

        //Was overflowing default MaxJsonLength
        //return Json(data, JsonRequestBehavior.AllowGet);
        return new JsonResult() { Data = data, JsonRequestBehavior = JsonRequestBehavior.AllowGet, MaxJsonLength = Int32.MaxValue };
      }
    }

    public JsonResult TaskInfo(string taskId)
    {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
      {
        var data =
          (from tasks in db.FactTasks
           join jobs in db.DimJobs
             on tasks.JobId equals jobs.JobId
           where tasks.TaskId.ToString() == taskId
           select new
           {
             TotalWaiting = tasks.TotalWaitingTime,
             TotalTransfer = tasks.TotalTransferTime,
             TotalRuntime = tasks.TotalRuntime,
             StartDate = tasks.StartTime
           }).OrderByDescending(s => s.StartDate).ToList();

        return Json(data, JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult GetSlaves(DateTime? start = null, DateTime? end = null, string userName = null, string slaveId = null)
    {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
      {
        var data =
          (from slaves in db.FactClientInfos
           join users in db.DimUsers
            on slaves.UserId equals users.UserId
           join clients in db.DimClients
            on slaves.ClientId equals clients.Id
           where (!start.HasValue || slaves.Time >= start) &&
           (!end.HasValue || slaves.Time < end) &&
           (string.IsNullOrEmpty(userName) || users.Name == userName) &&
           (string.IsNullOrEmpty(slaveId) || slaves.ClientId.ToString() == slaveId)
           select new
           {
             ClientName = clients.Name,
             SlaveID = slaves.ClientId,
           }).GroupBy(s => s.SlaveID).ToList();
        //Was overflowing default MaxJsonLength
        //return Json(data, JsonRequestBehavior.AllowGet);
        return new JsonResult() { Data = data, JsonRequestBehavior = JsonRequestBehavior.AllowGet, MaxJsonLength = Int32.MaxValue };
      }
    }

    public JsonResult SlaveInfo(string slaveId, DateTime? start = null, DateTime? end = null, string userName = null) {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString))
      {
        var data =
          (from slaves in db.FactClientInfos
           join users in db.DimUsers
            on slaves.UserId equals users.UserId
           join clients in db.DimClients
            on slaves.ClientId equals clients.Id
           where  slaves.ClientId.ToString() == slaveId && 
           (!start.HasValue || slaves.Time >= start) &&
           (!end.HasValue || slaves.Time < end) &&
           (string.IsNullOrEmpty(userName) || users.Name == userName)
           select new
           {
             ClientName = clients.Name,
             SlaveID = slaves.ClientId,
             Time = slaves.Time,
             UsedCores = slaves.NumUsedCores,
             TotalCores = slaves.NumTotalCores,
             UsedMemory = slaves.UsedMemory,
             TotalMemory = slaves.TotalMemory,
             CPUUtilization = slaves.CpuUtilization
           }).OrderByDescending(s => s.Time).ToList();

        return Json(data, JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult JobProgress(string jobId) {
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString)) {
        var data =
          (from tasks in db.FactTasks
           where tasks.JobId.ToString() == jobId
           select new
           {
             State = tasks.TaskState
           }).ToList();

        Dictionary<string, int> JobTasks = new Dictionary<string,int>();
        JobTasks.Add("Wait",0);
        JobTasks.Add("Transfer", 0);
        JobTasks.Add("Calculate", 0);
        JobTasks.Add("Finish", 0);
        JobTasks.Add("Error", 0);

        foreach(var record in data) {
          switch (record.State) {
            case TaskState.Waiting:
              JobTasks["Wait"] += 1;
              break;
            case TaskState.Transferring:
              JobTasks["Transfer"] += 1;
              break;
            case TaskState.Calculating:
              JobTasks["Calculate"] += 1;
              break;
            case TaskState.Finished:
              JobTasks["Finish"] += 1;
              break;
            case TaskState.Aborted:
            case TaskState.Failed:
              JobTasks["Error"] += 1;
              break;
          }
        }

        return Json(JobTasks, JsonRequestBehavior.AllowGet);
      }
    }

    public JsonResult UserTasks(string taskState) {
      TaskState? state = GetTaskState(taskState);
      List<KeyValuePair<string, int>> numberTasksByUser = new List<KeyValuePair<string,int>>();
      using (var db = new HiveDataContext(Settings.Default.HeuristicLab_Hive_LinqConnectionString)) {
        var userTasks = (from tasks in db.FactTasks
                         join jobs in db.DimJobs
                           on tasks.JobId equals jobs.JobId
                         where tasks.TaskState == state
                         select new
                         {
                           UserID = jobs.UserId,
                           UserName = jobs.UserName,
                           State = tasks.TaskState
                         }).GroupBy(j => j.UserID).ToList();

        foreach (var user in userTasks) {
          numberTasksByUser.Add(new KeyValuePair<string, int>(user.First().UserName, user.Count()));
        }
        return Json(numberTasksByUser, JsonRequestBehavior.AllowGet);
      }
    }

    private static TaskState? GetTaskState(string taskState) {
      TaskState? state;
      switch (taskState)
      {
        case "Aborted":
          state = TaskState.Aborted;
          break;
        case "Calculating":
          state = TaskState.Calculating;
          break;
        case "Failed":
          state = TaskState.Failed;
          break;
        case "Finished":
          state = TaskState.Finished;
          break;
        case "Offline":
          state = TaskState.Offline;
          break;
        case "Paused":
          state = TaskState.Paused;
          break;
        case "Transferring":
          state = TaskState.Transferring;
          break;
        case "Waiting":
          state = TaskState.Waiting;
          break;
        default :
          state = null;
          break;
      }
      return state;
    }

    private static IOrderedQueryable<IGrouping<DateTime, FactClientInfo>> GetClientFacts(HiveDataContext db, DateTime? start = null, DateTime? end = null) {
      start = start ?? DateTime.Now - DefaultDuration;
      end = end ?? DateTime.Now;

      return from ci in db.FactClientInfos
             where ci.Time >= start && ci.Time < end
             group ci by ci.Time into timeGroup
             orderby timeGroup.Key
             select timeGroup;
    }

    private static IEnumerable<IEnumerable<object[]>> CreateSeriesData<T>(IEnumerable<T> data, Func<T, DateTime> timeSelector, params Func<T, object>[] seriesSelectors) {
      return seriesSelectors.Select(selector =>
        data.Select(x => new[] {
          timeSelector(x).ToUniversalTime().ToUnixTimestamp(),
          selector(x)
        })
      );
    }

    private static DateTime GetRecentDate(HiveDataContext db)
    {
      var mostRecent = from dates in db.FactClientInfos
        group dates by dates.Time into startDate
        select startDate.OrderByDescending(t=>t.Time).FirstOrDefault();
      return mostRecent.Max(t=>t.Time);
    }
  }

  public static class DateTimeExtensions {
    public static long ToUnixTimestamp(this DateTime dateTime) {
      var duration = dateTime - new DateTime(1970, 1, 1, 0, 0, 0);

      return (long)duration.TotalMilliseconds;
    }
  }
}