source: branches/3.3-Hive/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3/HiveExperiment.cs @ 4368

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Linq;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Optimization;
using System.Drawing;
using HeuristicLab.Collections;
using System.Collections.Generic;
using HeuristicLab.Hive.Contracts.BusinessObjects;
using System.IO;
using HeuristicLab.Persistence.Default.Xml;
using HeuristicLab.PluginInfrastructure;
using System.Reflection;
using HeuristicLab.Hive.Contracts.Interfaces;
using HeuristicLab.Hive.Contracts;
using System.Threading;
using HeuristicLab.Tracing;
using HeuristicLab.Hive.JobBase;
using System.Diagnostics;
using System.Collections;
using System.ServiceModel;
using HeuristicLab.Hive.Contracts.ResponseObjects;
using HeuristicLab.Hive.Experiment.Properties;
using System.ComponentModel;
using HeuristicLab.Hive.Experiment.Jobs;

namespace HeuristicLab.Hive.Experiment {
  /// <summary>
  /// An experiment which contains multiple batch runs of algorithms.
  /// </summary>
  [Item(itemName, itemDescription)]
  [StorableClass]
  public class HiveExperiment : NamedItem, IExecutable {
    private const string itemName = "Hive Experiment";
    private const string itemDescription = "A runner for a single experiment, which's algorithms are executed in the Hive.";
    private const int resultPollingIntervalMs = 5000;
    private const int snapshotPollingIntervalMs = 1000;
    private const int maxSnapshotRetries = 20;
    private object locker = new object();

    private System.Timers.Timer timer;
    private bool pausePending, stopPending;

    // ensure that only 2 threads can fetch jobresults simultaniously
    private Semaphore fetchJobSemaphore = new Semaphore(2, 2);

    private static object pendingOptimizerMappingsLocker = new object();

    private bool stopResultsPollingPending = false;

    private Thread resultPollingThread;

    private bool isPollingResults;
    public bool IsPollingResults {
      get { return isPollingResults; }
      private set {
        if (isPollingResults != value) {
          isPollingResults = value;
          OnIsPollingResultsChanged();
        }
      }
    }

    public IEnumerable<string> ResourceGroups {
      get {
        if (!string.IsNullOrEmpty(resourceIds)) {
          return resourceIds.Split(';');
        } else {
          return new List<string>();
        }
      }
    }

    #region Storable Properties
    [Storable]
    private DateTime lastUpdateTime;
    
    [Storable]
    private string resourceIds;
    public string ResourceIds {
      get { return resourceIds; }
      set {
        if (resourceIds != value) {
          resourceIds = value;
          OnResourceIdsChanged();
        }
      }
    }

    [Storable]
    private HeuristicLab.Optimization.Experiment experiment;
    public HeuristicLab.Optimization.Experiment Experiment {
      get { return experiment; }
      set {
        if (experiment != value) {
          experiment = value;
          OnExperimentChanged();
        }
      }
    }

    [Storable]
    private ILog log;
    public ILog Log {
      get { return log; }
    }

    [Storable]
    private Core.ExecutionState executionState;
    public ExecutionState ExecutionState {
      get { return executionState; }
      private set {
        if (executionState != value) {
          executionState = value;
          OnExecutionStateChanged();
        }
      }
    }

    [Storable]
    private TimeSpan executionTime;
    public TimeSpan ExecutionTime {
      get { return executionTime; }
      private set {
        if (executionTime != value) {
          executionTime = value;
          OnExecutionTimeChanged();
        }
      }
    }

    [Storable]
    private IJob rootJob;
    public IJob RootJob {
      get { return rootJob; }
      set {
        if (rootJob != value) {
          rootJob = value;
          OnRootJobChanged();
        }
      }
    }

    private JobItem rootJobItem;
    public JobItem RootJobItem {
      get { return rootJobItem; }
      set {
        if (rootJobItem != null) {
          DeregisterRootJobItemEvents();
        }
        if (rootJobItem != value) {
          rootJobItem = value;
          RegisterRootJobItemEvents();
          OnRootJobItemChanged();
        }
      }
    }

    private void RegisterRootJobItemEvents() {
      rootJobItem.FinalResultAvailable += new EventHandler(rootJobItem_FinalResultAvailable);
    }

    private void DeregisterRootJobItemEvents() {
      rootJobItem.FinalResultAvailable -= new EventHandler(rootJobItem_FinalResultAvailable);
    }

    #endregion

    [StorableConstructor]
    public HiveExperiment(bool deserializing)
      : base(deserializing) {
    }

    public HiveExperiment()
      : base(itemName, itemDescription) {
      this.ResourceIds = HeuristicLab.Hive.Experiment.Properties.Settings.Default.ResourceIds;
      this.log = new Log();
      pausePending = stopPending = false;
      isPollingResults = false;
      InitTimer();
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      LogMessage("I am beeing cloned");
      HiveExperiment clone = (HiveExperiment)base.Clone(cloner);
      clone.resourceIds = this.resourceIds;
      clone.experiment = (HeuristicLab.Optimization.Experiment)cloner.Clone(experiment);
      clone.executionState = this.executionState;
      clone.executionTime = this.executionTime;
      clone.log = (ILog)cloner.Clone(log);
      clone.stopPending = this.stopPending;
      clone.pausePending = this.pausePending;
      clone.lastUpdateTime = this.lastUpdateTime;
      clone.isPollingResults = this.isPollingResults;
      clone.rootJob = (IJob)cloner.Clone(this.rootJob);
      return clone;
    }

    [StorableHook(HookType.AfterDeserialization)]
    private void AfterDeserialization() {
      InitTimer();
      this.IsPollingResults = false;
      this.stopResultsPollingPending = false;
      RegisterEvents();
      LogMessage("I was deserialized.");
    }

    private void RegisterEvents() {
      RootJobChanged += new EventHandler(HiveExperiment_RootJobChanged);
    }

    private void DeRegisterEvents() {
      RootJobChanged -= new EventHandler(HiveExperiment_RootJobChanged);
    }

    #region Execution Time Timer
    private void InitTimer() {
      timer = new System.Timers.Timer(100);
      timer.AutoReset = true;
      timer.Elapsed += new System.Timers.ElapsedEventHandler(timer_Elapsed);
    }

    private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) {
      DateTime now = DateTime.Now;
      ExecutionTime += now - lastUpdateTime;
      lastUpdateTime = now;
    }
    #endregion

    #region IExecutable Methods
    public void Pause() {
      throw new NotSupportedException();
    }

    public void Prepare() {
      if (experiment != null) {
        StopResultPolling();
        experiment.Prepare();
        this.ExecutionState = Core.ExecutionState.Prepared;
        OnPrepared();
      }
    }

    public void Start() {
      OnStarted();
      ExecutionTime = new TimeSpan();
      lastUpdateTime = DateTime.Now;
      this.ExecutionState = Core.ExecutionState.Started;

      Thread t = new Thread(() => {
        using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
          try {
            RootJobItem = ToJobItem(RootJob);

            IEnumerable<string> groups = ResourceGroups;
            SerializedJob serializedJob = RootJobItem.ToSerializedJob();
            ResponseObject<JobDto> response = service.Obj.AddJobWithGroupStrings(serializedJob, groups);

            if (response.StatusMessage != ResponseStatus.Ok) {
              throw new Exception(response.StatusMessage.ToString());
            } else {
              RootJobItem.JobDto = response.Obj;
              LogMessage(RootJobItem.JobDto.Id, "Job sent to Hive");

              StartResultPolling();
            }
          }
          catch (Exception e) {
            LogMessage("Error: Starting HiveExperiment failed: " + e.Message);
            this.ExecutionState = Core.ExecutionState.Stopped;
            OnStopped();
          }
        }
      });
      t.Start();
    }

    public void Stop() {
      if (IsPollingResults)
        StopResultPolling();
      this.ExecutionState = Core.ExecutionState.Stopped;
      OnStopped();
    }
    #endregion
    
    #region Job Management
    public void RefreshJobTree() {
      this.RootJob = CreateJobTree(this.Experiment);
    }
    
    private IJob CreateJobTree(IOptimizer optimizer) {
      IJob job = null;
      if (optimizer is HeuristicLab.Optimization.Experiment) {
        HeuristicLab.Optimization.Experiment exp = (HeuristicLab.Optimization.Experiment)optimizer;
        ExperimentJob expJob = new ExperimentJob(exp);
        foreach (IOptimizer opt in exp.Optimizers) {
          expJob.AddChildJob(CreateJobTree(opt));
        }
        job = expJob;
      } else if (optimizer is BatchRun) {
        job = new BatchRunJob(optimizer);
      } else {
        job = new OptimizerJob(optimizer);
      }
      return job;
    }

    private JobItem ToJobItem(IJob j) {
      return new JobItem() {
        Job = j,
        JobDto = new JobDto() {
          State = JobState.Offline,
          PluginsNeeded = HivePluginInfoDto.FindPluginsNeeded(j.GetType())
        }
      };
    }

    /// <summary>
    /// Updates all JobItems with the results
    /// if one is finished, the serialized job is downloaded and updated
    /// </summary>
    /// <param name="jobResultList"></param>
    private void UpdateJobItems(JobResultList jobResultList) {
      if (RootJobItem == null) {
        var rootResults = jobResultList.Where(res => !res.ParentJobId.HasValue);
        if (rootResults.Count() > 0) {
          RootJobItem = new JobItem(rootResults.First());
        } else {
          LogMessage("Error: Could not find JobResult for RootJobItem");
          return;
        }
      }

      using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
        foreach (JobResult jobResult in jobResultList) {
          JobItem jobItem = FindJobItem(RootJobItem, jobResult.Id);
          if (jobItem != null) {
            jobItem.UpdateJob(jobResult);
            if (jobItem.JobDto.State == JobState.Finished && !jobItem.IsFinalResultAvailable) {
              // job is finished but the final result was not yet downloaded
              SerializedJob serializedJob = service.Obj.GetLastSerializedResult(jobResult.Id).Obj;
              jobItem.Job = XmlParser.Deserialize<IJob>(new MemoryStream(serializedJob.SerializedJobData));
              UpdateChildJobs(jobItem);
              this.Experiment.Runs.AddRange(((OptimizerJob)jobItem.Job).Optimizer.Runs);
              //ReplaceOptimizerInExperiment(jobItem.JobDto.Id, jobItem.Job);
            }
          } else {
            // job does not yet exist locally
            JobItem parentJobItem = FindJobItem(RootJobItem, jobResult.ParentJobId.Value);
            if (parentJobItem != null) {
              LogMessage(jobResult.Id, "Creating JobItem");
              SerializedJob serializedJob = service.Obj.GetLastSerializedResult(jobResult.Id).Obj;
              JobItem newJobItem = new JobItem();
              newJobItem.JobDto = serializedJob.JobInfo;
              newJobItem.Job = XmlParser.Deserialize<IJob>(new MemoryStream(serializedJob.SerializedJobData));
              parentJobItem.AddChildJob(newJobItem);
              if (newJobItem.JobDto.State == JobState.Finished && newJobItem.IsFinalResultAvailable) {
                this.Experiment.Runs.AddRange(((OptimizerJob)newJobItem.Job).Optimizer.Runs);
                UpdateChildJobs(newJobItem);
                //ReplaceOptimizerInExperiment(jobItem.JobDto.Id, jobItem.Job);
              }
            } else {
              LogMessage("Error: Could not update JobResult for " + jobResult.Id);
            }
          }
        }
      }
    }

    /// <summary>
    /// Updates the ChildJobItems of a JobItem according to the IJob.ChildJobs of JobItem.Job (pretty confusing, right)
    /// </summary>
    /// <param name="jobItem"></param>
    private void UpdateChildJobs(JobItem jobItem) {
      List<JobItem> newJobItems = new List<JobItem>();
      foreach (IJob job in jobItem.Job.ChildJobs) {
        if (!jobItem.ReplaceChildJob(job)) {
          newJobItems.Add(ToJobItem(job));
        }
      }
      foreach (JobItem item in newJobItems) {
        jobItem.AddChildJob(item);
      }
    }

    private JobItem FindJobItem(JobItem parentJobItem, Guid jobId) {
      if (parentJobItem.JobDto.Id == jobId) {
        return parentJobItem;
      } else {
        foreach (JobItem child in parentJobItem.ChildJobItems) {
          JobItem result = FindJobItem(child, jobId);
          if (result != null)
            return result;
        }
      }
      return null;
    }

    private void JobItem_JobStateChanged(object sender, EventArgs e) {
      JobItem jobItem = (JobItem)sender;

      Thread t = new Thread(() => {
        try {
          if (jobItem.JobDto.State == JobState.Finished) {
            FetchAndUpdateJob(jobItem.JobDto.Id);
          } else if (jobItem.JobDto.State == JobState.Failed) {
            
          }
        }
        catch (Exception ex) {
          LogMessage("JobItem_JobStateChanged failed badly: " + ex.Message);
        }
      });
      t.Start();
    }

    /// <summary>
    /// Fetches the finished job from the server and updates the jobItem
    /// </summary>
    private void FetchAndUpdateJob(Guid jobId) {
      bool tryagain = false;
      LogMessage(jobId, "FetchAndUpdateJob started");
      if (fetchJobSemaphore.WaitOne(new TimeSpan(0, 2, 0))) {

        try {
          using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
            ResponseObject<SerializedJob> jobResponse = service.Obj.GetLastSerializedResult(jobId);
            IJob restoredObject = XmlParser.Deserialize<IJob>(new MemoryStream(jobResponse.Obj.SerializedJobData));
            IOptimizer restoredOptimizer = ((OptimizerJob)restoredObject).Optimizer;
            LogMessage(jobId, "FetchAndUpdateJob ended");
          }
        }
        catch (Exception e) {
          LogMessage(jobId, "FetchAndUpdateJob failed: " + e.Message + ". Will try again!");
          tryagain = true;
        }
        finally {
          fetchJobSemaphore.Release();
        }
      } else {
        LogMessage(jobId, "FetchAndUpdateJob timed out. Will try again!");
        tryagain = true;
      }

      if (tryagain) {
        FetchAndUpdateJob(jobId);
      }
    }

    public void AbortJob(Guid jobId) {
      using (Disposable<IClientFacade> service = ServiceLocator.Instance.ClientFacadePool.GetService()) {
        Response response = service.Obj.AbortJob(jobId);
        LogMessage(jobId, "Aborting Job: " + response.StatusMessage);
      }
    }
    #endregion

    #region Result Polling
    public void StartResultPolling() {
      this.stopResultsPollingPending = false;
      this.IsPollingResults = true;
      resultPollingThread = new Thread(RunResultPollingThread);
      if (resultPollingThread.ThreadState != System.Threading.ThreadState.Running)
        resultPollingThread.Start();
    }

    public void StopResultPolling() {
      this.stopResultsPollingPending = true;
      if (resultPollingThread != null) {
        if (resultPollingThread.ThreadState == System.Threading.ThreadState.WaitSleepJoin) {
          resultPollingThread.Interrupt();
        }
        resultPollingThread.Join();
      }
      this.stopResultsPollingPending = false;
    }

    private void RunResultPollingThread() {
      try {
        do {
          if (RootJobItem.JobDto.State != JobState.Finished) {
            using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
              LogMessage("Polling results");
              try {
                ResponseObject<JobResultList> response = service.Obj.GetChildJobResults(RootJobItem.JobDto.Id, true, true);
                if (response.StatusMessage == ResponseStatus.Ok) {
                  JobResultList jobItemList = response.Obj;
                  UpdateJobItems(jobItemList);

                  LogMessage("Polling successfully finished");
                } else {
                  throw new Exception(response.StatusMessage.ToString());
                }
              }
              catch (Exception e) {
                LogMessage("Polling results failed: " + e.Message);
              }
            }
            Thread.Sleep(resultPollingIntervalMs);
          } else {
            // all the jobs have been sent to hive, but non are to query any more (all finished or failed)
            this.stopResultsPollingPending = true;
          }
        } while (!this.stopResultsPollingPending);
      }
      catch (ThreadInterruptedException) {
        // thread has been interuppted
      }
      catch (Exception e) {
        LogMessage("Result Polling Thread failed badly: " + e.Message);
      }
      finally {
        this.IsPollingResults = false;
      }
    }

    #endregion

    #region Snapshots

    //private void UpdateSnapshot(ResponseObject<SerializedJob> response) {
    //  JobItem jobItem = jobItems.Single(x => x.JobDto.Id == response.Obj.JobInfo.Id);
    //  jobItem.LatestSnapshot = response;
    //}

    //public void RequestSnapshot(Guid jobId) {
    //  Thread t = new Thread(() => {
    //    IClientFacade clientFacade = null;
    //    try {
    //      clientFacade = CreateStreamedClientFacade();

    //      ResponseObject<SerializedJob> response;
    //      int retryCount = 0;

    //      Response snapShotResponse = clientFacade.RequestSnapshot(jobId);
    //      if (snapShotResponse.StatusMessage == ResponseStatus.RequestSnapshot_JobIsNotBeeingCalculated) {
    //        // job already finished
    //        Logger.Debug("HiveExperiment: Abort - GetLastResult(false)");
    //        response = clientFacade.GetLastSerializedResult(jobId, false, false);
    //        Logger.Debug("HiveExperiment: Abort - Server: " + response.StatusMessage);
    //      } else {
    //        // server sent snapshot request to client
    //        // poll until snapshot is ready
    //        do {
    //          Thread.Sleep(snapshotPollingIntervalMs);
    //          Logger.Debug("HiveExperiment: Abort - GetLastResult(true)");
    //          response = clientFacade.GetLastSerializedResult(jobId, false, true);
    //          Logger.Debug("HiveExperiment: Abort - Server: " + response.StatusMessage);
    //          retryCount++;
    //          // loop while
    //          // 1. problem with communication with server
    //          // 2. job result not yet ready
    //        } while (
    //          (retryCount < maxSnapshotRetries) && (
    //          response.StatusMessage == ResponseStatus.GetLastSerializedResult_JobResultNotYetThere)
    //          );
    //      }
    //      if (response.StatusMessage == ResponseStatus.Ok) {
    //        LogMessage(jobId, "Snapshot polling successfull for job " + jobId);
    //        //UpdateSnapshot(response);
    //      } else {
    //        LogMessage(jobId, "Error: Polling of Snapshot failed for job " + jobId + ": " + response.StatusMessage);
    //      }
    //    }
    //    catch (Exception e) {
    //      LogMessage("RequestSnapshot Thread failed badly: " + e.Message);
    //      Logger.Error("RequestSnapshot Thread failed badly: " + e.Message);
    //    }
    //    finally {
    //      ServiceLocator.DisposeClientFacade(clientFacade);
    //    }
    //  });
    //  t.Start();
    //}

    //void JobItem_SnapshotRequestedStateChanged(object sender, EventArgs e) {
    //  JobItem jobItem = (JobItem)sender;
    //  if (jobItem.SnapshotRequestedState == SnapshotRequestedState.Requested) {
    //    RequestSnapshot(jobItem.JobDto.Id);
    //  }
    //}

    #endregion

    #region Eventhandler

    public event EventHandler ExecutionTimeChanged;
    private void OnExecutionTimeChanged() {
      EventHandler handler = ExecutionTimeChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler ExecutionStateChanged;
    private void OnExecutionStateChanged() {
      LogMessage("ExecutionState changed to " + executionState.ToString());
      EventHandler handler = ExecutionStateChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;

    public event EventHandler Started;
    private void OnStarted() {
      LogMessage("Started");
      timer.Start();
      EventHandler handler = Started;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler Stopped;
    private void OnStopped() {
      timer.Stop();
      LogMessage("Stopped");
      EventHandler handler = Stopped;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler Paused;
    private void OnPaused() {
      timer.Stop();
      LogMessage("Paused");
      EventHandler handler = Paused;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler Prepared;
    protected virtual void OnPrepared() {
      LogMessage("Prepared");
      EventHandler handler = Prepared;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler ResourceIdsChanged;
    protected virtual void OnResourceIdsChanged() {
      EventHandler handler = ResourceIdsChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler ExperimentChanged;
    protected virtual void OnExperimentChanged() {
      LogMessage("Experiment changed");
      EventHandler handler = ExperimentChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler RootJobChanged;
    private void OnRootJobChanged() {
      EventHandler handler = RootJobChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler RootJobItemChanged;
    private void OnRootJobItemChanged() {
      EventHandler handler = RootJobItemChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler IsResultsPollingChanged;
    private void OnIsPollingResultsChanged() {
      if (this.IsPollingResults) {
        LogMessage("Results Polling Started");
        timer.Start();
      } else {
        LogMessage("Results Polling Stopped");
        timer.Stop();
      }
      EventHandler handler = IsResultsPollingChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    void HiveExperiment_RootJobChanged(object sender, EventArgs e) {
      this.Experiment = (Optimization.Experiment)((ExperimentJob)rootJobItem.Job).Optimizer;
    }

    void rootJobItem_FinalResultAvailable(object sender, EventArgs e) {
      this.Experiment = (Optimization.Experiment)((ExperimentJob)rootJobItem.Job).Optimizer;
    }
    #endregion
    
    #region Logging
    private void LogMessage(string message) {
      // HeuristicLab.Log is not Thread-Safe, so lock on every call
      lock (locker) {
        log.LogMessage(message);
        Logger.Debug(message);
      }
    }

    private void LogMessage(Guid jobId, string message) {
      //GetJobItemById(jobId).LogMessage(message);
      LogMessage(message + " (jobId: " + jobId + ")");
    }

    #endregion

    /// <summary>
    /// Downloads the root job from hive and sets the experiment, rootJob and rootJobItem
    /// </summary>
    public void LoadExperimentFromHive() {
      using (Disposable<IClientFacade> service = ServiceLocator.Instance.StreamedClientFacadePool.GetService()) {
        ResponseObject<SerializedJob> response = service.Obj.GetLastSerializedResult(RootJobItem.JobDto.Id);
        IJob job = XmlParser.Deserialize<IJob>(new MemoryStream(response.Obj.SerializedJobData));
        RootJob = job;
        RootJobItem.JobDto = response.Obj.JobInfo;
        this.Experiment = (Optimization.Experiment)((OptimizerJob)job).Optimizer;
        if (rootJobItem.JobDto.State == JobState.Finished) {
          // set execution time and finish up
        } else if (rootJobItem.JobDto.State != JobState.Aborted && rootJobItem.JobDto.State != JobState.Failed) {
          this.ExecutionState = Core.ExecutionState.Started;
        }
      }
    }
  }
}