source: branches/3.3-HiveMigration/sources/HeuristicLab.Hive/HeuristicLab.Hive.Experiment/3.3/HiveExperiment.cs @ 4119

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Linq;
using HeuristicLab.Common;
using HeuristicLab.Core;
using HeuristicLab.Persistence.Default.CompositeSerializers.Storable;
using HeuristicLab.Optimization;
using System.Drawing;
using HeuristicLab.Collections;
using System.Collections.Generic;
using HeuristicLab.Hive.Contracts.BusinessObjects;
using System.IO;
using HeuristicLab.Persistence.Default.Xml;
using HeuristicLab.PluginInfrastructure;
using System.Reflection;
using HeuristicLab.Hive.Contracts.Interfaces;
using HeuristicLab.Hive.Contracts;
using System.Threading;
using HeuristicLab.Tracing;
using HeuristicLab.Hive.JobBase;
using System.Diagnostics;
using System.Collections;

namespace HeuristicLab.Hive.Experiment {
  /// <summary>
  /// An experiment which contains multiple batch runs of algorithms.
  /// </summary>
  [Item(itemName, itemDescription)]
  [Creatable("Testing & Analysis")]
  [StorableClass]
  public class HiveExperiment : NamedItem, IExecutable {
    private const string itemName = "Hive Experiment";
    private const string itemDescription = "An experiment which contains multiple batch runs of algorithms which are executed in the Hive.";
    private const int resultPollingIntervalMs = 10000;

    private const int maxSnapshotRetries = 20;

    [Storable]
    private IDictionary<Guid, IOptimizer> sentOptimizers = null;

    private object locker = new object();
    private DateTime startTime;

    [Storable]
    private string serverUrl;
    public string ServerUrl {
      get { return serverUrl; }
      set { serverUrl = value; OnServerUrlChanged(); }
    }

    [Storable]
    private string resourceIds;
    public string ResourceIds {
      get { return resourceIds; }
      set { resourceIds = value; OnResourceIdsChanged(); }
    }

    [Storable]
    private HeuristicLab.Optimization.Experiment experiment;
    public HeuristicLab.Optimization.Experiment Experiment {
      get { return experiment; }
      set { experiment = value; OnExperimentChanged(); }
    }

    [StorableConstructor]
    public HiveExperiment(bool deserializing)
      : base(deserializing) {
    }

    public HiveExperiment()
      : base(itemName, itemDescription) {
      this.ServerUrl = HeuristicLab.Hive.Experiment.Properties.Settings.Default.HiveServerUrl;
      this.ResourceIds = HeuristicLab.Hive.Experiment.Properties.Settings.Default.ResourceIds;
    }

    public override IDeepCloneable Clone(Cloner cloner) {
      HiveExperiment clone = (HiveExperiment)base.Clone(cloner);
      clone.resourceIds = this.resourceIds;
      clone.serverUrl = this.serverUrl;
      clone.experiment = (HeuristicLab.Optimization.Experiment)cloner.Clone(experiment);
      clone.executionState = this.executionState;
      clone.executionTime = this.executionTime;
      clone.sentOptimizers = new Dictionary<Guid, IOptimizer>();
      foreach (var pair in this.sentOptimizers)
        clone.sentOptimizers[pair.Key] = (IOptimizer)cloner.Clone(pair.Value);
      return clone;
    }

    public IEnumerable<string> ResourceGroups {
      get {
        if (!string.IsNullOrEmpty(resourceIds)) {
          return resourceIds.Split(';');
        } else {
          return new List<string>();
        }
      }
    }
    #region IExecutable Members

    [Storable]
    private Core.ExecutionState executionState;
    public ExecutionState ExecutionState {
      get { return executionState; }
      private set {
        if (executionState != value) {
          executionState = value;
          OnExecutionStateChanged();
        }
      }
    }

    [Storable]
    private TimeSpan executionTime;
    public TimeSpan ExecutionTime {
      get { return executionTime; }
      private set {
        executionTime = value;
        OnExecutionTimeChanged();
      }
    }

    public void Pause() {
      throw new NotSupportedException();
    }

    public void Prepare() {
      // [chn] needed?
      if (experiment != null) {
        experiment.Prepare();
        OnPrepared();
      }
    }

    public void Start() {
      this.startTime = DateTime.Now;
      this.ExecutionState = Core.ExecutionState.Started;
      IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(ServerUrl);

      sentOptimizers = new Dictionary<Guid, IOptimizer>();
      IEnumerable<string> groups = ResourceGroups;

      foreach (IOptimizer optimizer in Experiment.Optimizers) {
        SerializedJob serializedJob = CreateSerializedJob(optimizer);
        ResponseObject<JobDto> response = executionEngineFacade.AddJobWithGroupStrings(serializedJob, groups);
        sentOptimizers.Add(response.Obj.Id, optimizer);
        StartResultPollingThread(response.Obj);
      }
    }

    public void Stop() {
      // todo
    }

    private bool abortRequested;

    #endregion

    private SerializedJob CreateSerializedJob(IOptimizer optimizer) {
      IJob job = new OptimizerJob() {
        Optimizer = optimizer
      };

      // serialize job
      MemoryStream memStream = new MemoryStream();
      XmlGenerator.Serialize(job, memStream);
      byte[] jobByteArray = memStream.ToArray();
      memStream.Dispose();

      // find out which which plugins are needed for the given object
      List<HivePluginInfoDto> pluginsNeeded = (
        from p in GetDeclaringPlugins(optimizer.GetType())
        select new HivePluginInfoDto() {
          Name = p.Name,
          Version = p.Version
        }).ToList();

      JobDto jobDto = new JobDto() {
        CoresNeeded = 1, // [chn] how to determine real cores needed?
        PluginsNeeded = pluginsNeeded,
        State = State.Offline,
        MemoryNeeded = 0,
        UserId = Guid.Empty // [chn] set real userid here!
      };

      SerializedJob serializedJob = new SerializedJob() {
        JobInfo = jobDto,
        SerializedJobData = jobByteArray
      };

      return serializedJob;
    }

    private void StartResultPollingThread(JobDto job) {
      Thread t = new Thread(() => {
        IExecutionEngineFacade executionEngineFacade = ServiceLocator.CreateExecutionEngineFacade(ServerUrl);
        IJob restoredObject = null;

        do {
          Thread.Sleep(resultPollingIntervalMs);
          lock (locker) {
            if (abortRequested) return;

            ResponseObject<SerializedJob> response = executionEngineFacade.GetLastSerializedResult(job.Id, false, false);

            // loop while 
            // 1. the user doesn't request an abort 
            // 2. there is a problem with server communication (success==false)
            // 3. no result for the job is available yet (response.Obj==null)
            // 4. the result that we get from the server is a snapshot and not the final result
            if (response.Success && response.Obj != null && response.StatusMessage != ApplicationConstants.RESPONSE_JOB_RESULT_NOT_YET_HERE) {

              // [chn] todo: deserialization into interface is not possible! how to find out correct static type?
              restoredObject = XmlParser.Deserialize<IJob>(new MemoryStream(response.Obj.SerializedJobData));
            }
          }
        } while (restoredObject == null || restoredObject.ExecutionState != Core.ExecutionState.Stopped);

        // job retrieved... replace the existing optimizers with the finished one
        IOptimizer originalOptimizer = sentOptimizers[job.Id];
        IOptimizer restoredOptimizer = ((OptimizerJob)restoredObject).Optimizer;

        int originalOptimizerIndex = experiment.Optimizers.IndexOf(originalOptimizer);
        experiment.Optimizers[originalOptimizerIndex] = restoredOptimizer;
        sentOptimizers.Remove(job.Id);

        if (sentOptimizers.Count == 0) {
          // finished
          this.ExecutionState = Core.ExecutionState.Stopped;
          OnStopped();
        }
      });

      Logger.Debug("HiveEngine: Starting results-polling thread");
      t.Start();
    }


    #region Required Plugin Search
    /// <summary>
    /// Returns a list of plugins in which the type itself and all members 
    /// of the type are declared. Objectgraph is searched recursively.
    /// </summary>
    private IEnumerable<IPluginDescription> GetDeclaringPlugins(Type type) {
      HashSet<Type> types = new HashSet<Type>();
      FindTypes(type, types, "HeuristicLab.");
      return GetDeclaringPlugins(types);
    }

    /// <summary>
    /// Returns the plugins (including dependencies) in which the given types are declared
    /// </summary>
    private IEnumerable<IPluginDescription> GetDeclaringPlugins(IEnumerable<Type> types) {
      HashSet<IPluginDescription> plugins = new HashSet<IPluginDescription>();
      foreach (Type t in types) {
        FindDeclaringPlugins(ApplicationManager.Manager.GetDeclaringPlugin(t), plugins);
      }
      return plugins;
    }

    /// <summary>
    /// Finds the dependencies of the given plugin and adds it to the plugins hashset.
    /// Also searches the dependencies recursively.
    /// </summary>
    private void FindDeclaringPlugins(IPluginDescription plugin, HashSet<IPluginDescription> plugins) {
      if (!plugins.Contains(plugin)) {
        plugins.Add(plugin);
        foreach (IPluginDescription dependency in plugin.Dependencies) {
          FindDeclaringPlugins(dependency, plugins);
        }
      }
    }

    /// <summary>
    /// Recursively finds all types used in type which are in a namespace which starts with namespaceStart
    /// Be aware that search is not performed on attributes
    /// </summary>
    /// <param name="type">the type to be searched</param>
    /// <param name="types">found types will be stored there, needed in order to avoid duplicates</param>
    /// <param name="namespaceStart">only types from namespaces which start with this will be searched and added</param>
    private void FindTypes(Type type, HashSet<Type> types, string namespaceStart) {
      if (!types.Contains(type) && type.Namespace.StartsWith(namespaceStart)) {
        types.Add(type);

        // constructors
        foreach (ConstructorInfo info in type.GetConstructors()) {
          foreach (ParameterInfo paramInfo in info.GetParameters()) {
            FindTypes(paramInfo.ParameterType, types, namespaceStart);
          }
        }

        // interfaces
        foreach (Type t in type.GetInterfaces()) {
          FindTypes(t, types, namespaceStart);
        }

        // events
        foreach (EventInfo info in type.GetEvents()) {
          FindTypes(info.EventHandlerType, types, namespaceStart);
          FindTypes(info.DeclaringType, types, namespaceStart);
        }

        // properties
        foreach (PropertyInfo info in type.GetProperties()) {
          FindTypes(info.PropertyType, types, namespaceStart);
        }

        // fields
        foreach (FieldInfo info in type.GetFields()) {
          FindTypes(info.FieldType, types, namespaceStart);
        }

        // methods
        foreach (MethodInfo info in type.GetMethods()) {
          foreach (ParameterInfo paramInfo in info.GetParameters()) {
            FindTypes(paramInfo.ParameterType, types, namespaceStart);
          }
          FindTypes(info.ReturnType, types, namespaceStart);
        }
      }
    }
    #endregion

    #region Eventhandler

    public event EventHandler ExecutionTimeChanged;
    private void OnExecutionTimeChanged() {
      EventHandler handler = ExecutionTimeChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler ExecutionStateChanged;
    private void OnExecutionStateChanged() {
      EventHandler handler = ExecutionStateChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler<EventArgs<Exception>> ExceptionOccurred;

    public event EventHandler Started;
    private void OnStarted() {
      EventHandler handler = Started;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler Stopped;
    private void OnStopped() {
      EventHandler handler = Stopped;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler Paused;
    private void OnPaused() {
      EventHandler handler = Paused;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler Prepared;
    protected virtual void OnPrepared() {
      EventHandler handler = Prepared;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler ResourceIdsChanged;
    protected virtual void OnResourceIdsChanged() {
      EventHandler handler = ResourceIdsChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler ExperimentChanged;
    protected virtual void OnExperimentChanged() {
      EventHandler handler = ExperimentChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }

    public event EventHandler ServerUrlChanged;
    protected virtual void OnServerUrlChanged() {
      EventHandler handler = ServerUrlChanged;
      if (handler != null) handler(this, EventArgs.Empty);
    }


    #endregion
  }
}