Free cookie consent management tool by TermsFeed Policy Generator

Ignore:
Timestamp:
06/17/19 10:40:41 (5 years ago)
Author:
dpiringe
Message:

#2924:

  • some changes in CLIApplication.cs to reduce unnecessary allocation of string objects
  • renamed AppTest to ConsoleOptimizer and fixed race condition
  • replaced enum RunnerJob with class RunnerMessage for more control of saved data
  • changed usage of BinaryFormatter with HEAL.Attic, following types are now Storable:
    • ConsoleOptimizer
    • InspectApplication
    • ApplicationBase
    • ApplicationRunner
    • AssemblyInfo
    • Runner
    • UniPath
    • RunnerMessage
  • switched QuietMode from ApplicationRunner to IRunner
  • DockerRunnerHost can now automatically build docker images for linux and windows containers (also identifies which container type is active) -> removes the condition to have the image preinstalled
    • to achieve this, there are two new folders DockerLinuxBuild and DockerWindowsBuild included in build output, which include Dockerfiles to build images for linux and windows container
  • added NuGet package System.CodeDom to project HeuristicLab.Scripting-3.3
  • added method Send(RunnerMessage) to IRunnerHost and transferred methods Pause and Resume to IRunner
  • added internal reference of RunnerHost in Runner
  • added abstract method SendMessage(RunnerMessage) in RunnerHost which gets called from method Send(RunnerMessage)
  • because of a Google.Protobuf "bug", RunnerMessages cannot get serialized/deserialized directly on stream -> workaround with a byte array, which gets written and read
    • additionally, the length of the array gets sent first (length as integer -> 4 bytes)
    • static method in RunnerMessage to read a message from a stream
    • the method SendMessage(RunnerMessage) in RunnerHost implements this functionality
Location:
branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3
Files:
1 added
1 deleted
10 edited

Legend:

Unmodified
Added
Removed

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/ApplicationBase.cs

    r16993 r17013  
    2121
    2222
    23 using System;
     23using HEAL.Attic;
    2424
    2525namespace HeuristicLab.PluginInfrastructure {
    … …  
    2727  /// Abstract base implementation for the IApplication interface.
    2828  /// </summary>
    29   [Serializable]
     29  [StorableType("FB13839C-2EEC-4C6F-BEF3-A1CAE4467362")]
    3030  public abstract class ApplicationBase : IApplication {
    3131    /// <summary>

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/HeuristicLab.PluginInfrastructure-3.3.csproj

    r16993 r17013  
    1717  </PropertyGroup>
    1818  <ItemGroup>
    19     <PackageReference Include="System.CodeDom" Version="4.5.0" />
    20     <PackageReference Include="System.Drawing.Common" Version="4.5.1" />
    2119    <PackageReference Include="System.Runtime.Loader" Version="4.3.0" />
    22     <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.5.0" />
    23     <PackageReference Include="System.Configuration.ConfigurationManager" Version="4.5.0" />
    24     <PackageReference Include="System.Security.Cryptography.Primitives" Version="4.3.0" />
     20  </ItemGroup>
     21  <ItemGroup>
     22    <Reference Include="HEAL.Attic">
     23      <HintPath>..\..\bin\HEAL.Attic.dll</HintPath>
     24      <Private>true</Private>
     25    </Reference>
    2526  </ItemGroup>
    2627</Project>

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Interfaces/IRunner.cs

    r16993 r17013  
    33namespace HeuristicLab.PluginInfrastructure {
    44  public interface IRunner {
    5     /// <summary>
    6     /// Set this to true, if console output should be disabled.
    7     /// </summary>
    8     bool QuietMode { get; set; }
    9 
    105    /// <summary>
    116    /// Assemblies which the child process needs to load.
    … …  
    1712    /// </summary>
    1813    void Run();
     14
     15    void Pause();
     16    void Resume();
     17    void Cancel();
    1918  }
    2019}

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Interfaces/IRunnerHost.cs

    r16993 r17013  
    1212    /// </summary>
    1313    RunnerState State { get; }
     14
     15    /// <summary>
     16    /// Set this to true, if console output should be disabled.
     17    /// </summary>
     18    bool QuietMode { get; set; }
    1419
    1520    /// <summary>
    … …  
    2833
    2934    /// <summary>
    30     /// Method to pause the runner and the isolated IApplication.
     35    /// Method to send messages to the runner, which runs in child process.
    3136    /// </summary>
    32     void Pause();
    33 
    34     /// <summary>
    35     /// Method to resume a paused runner and the isolated IApplication.
    36     /// </summary>
    37     void Resume();
     37    /// <param name="runnerMessage">A object from a derived class of RunnerMessage.</param>
     38    void Send(RunnerMessage runnerMessage);
    3839  }
    3940}

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/ApplicationRunner.cs

    r16993 r17013  
    1 using System;
    2 using System.IO;
     1using HEAL.Attic;
    32
    43namespace HeuristicLab.PluginInfrastructure {
    5   [Serializable]
     4  [StorableType("612F98AF-E254-4C5E-BD41-75B4F1D9B96D")]
    65  public class ApplicationRunner : Runner {
    76    /// <summary>
    87    /// Arguments for the StartApplication.
    98    /// </summary>
     9    [Storable]
    1010    public ICommandLineArgument[] Args { get; set; }
    1111
    … …  
    1515    public IApplication StartApplication {
    1616      get {
    17         if (application == null) {
    18           using (var memStream = new MemoryStream(serializedStartApplication)) {
    19             application = (IApplication)formatter.Deserialize(memStream);
    20           }
     17        lock (locker) {
     18          if (application == null)
     19            application = (IApplication)new ProtoBufSerializer().Deserialize(serializedStartApplication);
     20          return application;
    2121        }
    22         return application;
    2322      }
    2423      set {
    25         using (var ms = new MemoryStream()) {
    26           formatter.Serialize(ms, value);
    27           serializedStartApplication = ms.ToArray();
     24        lock (locker) {
     25          serializedStartApplication = new ProtoBufSerializer().Serialize(value);
     26          application = value;
    2827        }
    2928      }
    … …  
    3130    // Encapsulated application is necessary, because it is not possible to
    3231    // instantly deserialize the application, before all assemblies are loaded.
     32    [Storable]
    3333    private byte[] serializedStartApplication = new byte[0];
     34
    3435    // cache application to prevent new instances every get call of StartApplication
    3536    private IApplication application;
     37    private object locker = new object();
    3638
    3739    protected override void Execute() {
    … …  
    3941    }
    4042
    41     protected override void OnRunnerJob(RunnerJob runnerJob) {
    42       switch (runnerJob) {
    43         case RunnerJob.Cancel: StartApplication.OnCancel(); break;
    44         case RunnerJob.Pause: StartApplication.OnPause(); break;
    45         case RunnerJob.Resume: StartApplication.OnResume(); break;
    46       }
     43    protected override void OnRunnerMessage(RunnerMessage message) {
     44      if (message is PauseRunnerMessage)
     45        StartApplication.OnPause();
     46      else if (message is ResumeRunnerMessage)
     47        StartApplication.OnResume();
     48      else if (message is CancelRunnerMessage)
     49        StartApplication.OnCancel();
    4750    }
    4851  }

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/AssemblyInfo.cs

    r16984 r17013  
    1 using System;
     1using HEAL.Attic;
    22
    33namespace HeuristicLab.PluginInfrastructure {
    4   [Serializable]
     4  [StorableType(StorableMemberSelection.AllProperties, "D1DC9C3E-16B7-464E-A377-35FA614195AD")]
    55  public class AssemblyInfo {
    66    /// <summary>

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/DockerRunnerHost.cs

    r16993 r17013  
    11using System.Diagnostics;
     2using System.IO;
     3using System.Threading.Tasks;
    24using HeuristicLab.PluginInfrastructure.Exceptions;
    35
    … …  
    810  public class DockerRunnerHost : RunnerHost {
    911    #region Constants
    10     private const string Docker = "docker";
    11     private const string ContainerStartup = "container run -i --rm ";
    12     private const string Mounting = @"--mount type=bind,source=/c/Users/,target=/Users ";
    13     private const string Image = "heuristiclab33:latest";
    14     private const string DockerExceptionMessage = "Docker is not running or image '" + Image + "' does not exists!";
     12    private readonly static string Docker = "docker";
     13    private readonly static string ContainerStartup = "container run -i --rm ";
     14    private readonly static string MountingLinux = @"--mount type=bind,source=/c/Users/,target=/Users ";
     15    private readonly static string MountingWindows = @"--mount type=bind,source=C:\Users\,target=C:\Users\ ";
     16    private readonly static string LinuxImage = "." + Path.DirectorySeparatorChar + "DockerLinuxBuild" + Path.DirectorySeparatorChar + "Dockerfile";
     17    private readonly static string WindowsImage = "." + Path.DirectorySeparatorChar + "DockerWindowsBuild" + Path.DirectorySeparatorChar + "Dockerfile";
     18    private readonly static string Image = "heuristiclab33:latest";
     19    private readonly static string DockerExceptionMessage = "Docker is not running!";
    1520    #endregion
    1621
    1722    #region Constructors
    18     public DockerRunnerHost(bool doDockerAvailableCheck = true)
    19       : base(Docker, ContainerStartup + Mounting + Image, null, null, null) {
    20       if (doDockerAvailableCheck && !IsDockerAvailable())
    21         throw new DockerException(DockerExceptionMessage);
     23    public DockerRunnerHost()
     24      : base(Docker, ContainerStartup + GetTargetOSMounting() + Image, null, null, null) {
    2225    }
    2326    #endregion
    2427
    2528    #region Helper
    26     private bool IsDockerAvailable() {
     29
     30    private static string GetTargetOSMounting() {
     31      Task<bool> win = Task.Run(() => BuildImage(WindowsImage));
     32      Task<bool> lin = Task.Run(() => BuildImage(LinuxImage));
     33      if (win.Result) return MountingWindows;
     34      else if (lin.Result) return MountingLinux;
     35      else throw new DockerException(DockerExceptionMessage);
     36    }
     37
     38    private static bool BuildImage(string pathToDockerfile) {
    2739      var process = new Process {
    2840        StartInfo = new ProcessStartInfo {
    2941          FileName = Docker,
    30           Arguments = "image inspect " + Image,
     42          Arguments = "image build -t " + Image +
     43            " -f " + Path.GetFullPath(pathToDockerfile) +
     44            " .",
    3145          UseShellExecute = false,
    3246          RedirectStandardOutput = true,
    … …  
    3751        EnableRaisingEvents = false
    3852      };
    39 
    4053      process.Start();
    4154      process.BeginOutputReadLine();

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/Runner.cs

    r16993 r17013  
    22using System.Collections.Generic;
    33using System.IO;
    4 using System.Runtime.Serialization;
    5 using System.Runtime.Serialization.Formatters.Binary;
    64using System.Threading;
     5using System.Threading.Tasks;
     6using HEAL.Attic;
    77
    88namespace HeuristicLab.PluginInfrastructure {
    9   [Serializable]
     9  [StorableType(StorableMemberSelection.MarkedOnly, "D93CBE04-9847-417A-AAB5-7FBCA6A32247")]
    1010  public abstract class Runner : IRunner {
    1111
    1212    #region Vars
    13     [NonSerialized]
    1413    private Thread listener;
    15 
    16     [NonSerialized]
    17     private Thread executor;
    18 
    19     [NonSerialized]
    20     protected static IFormatter formatter = new BinaryFormatter();
    2114    #endregion
    2215
    2316    #region Properties
    24     public bool QuietMode { get; set; }
     17    [Storable]
     18    public IEnumerable<AssemblyInfo> AssembliesToLoad { get; set; }
    2519
    26     public IEnumerable<AssemblyInfo> AssembliesToLoad { get; set; }
     20    internal RunnerHost Host { get; set; }
    2721    #endregion
    2822
     23    public void Pause() {
     24      var message = new PauseRunnerMessage();
     25      if (Host != null) Host.Send(message);
     26      else OnRunnerMessage(message);
     27    }
    2928
    30     public static void Serialize(IRunner runner, Stream stream) => formatter.Serialize(stream, runner);
    31     public static IRunner Deserialize(Stream stream = null) => (IRunner)formatter.Deserialize(stream);
    32     public static void Pause(Stream stream) => formatter.Serialize(stream, RunnerJob.Pause);
    33     public static void Resume(Stream stream) => formatter.Serialize(stream, RunnerJob.Resume);
    34     public static void Cancel(Stream stream) => formatter.Serialize(stream, RunnerJob.Cancel);
     29    public void Resume() {
     30      var message = new ResumeRunnerMessage();
     31      if (Host != null) Host.Send(message);
     32      else OnRunnerMessage(message);
     33    }
     34
     35    public void Cancel() {
     36      var message = new CancelRunnerMessage();
     37      if (Host != null) Host.Send(message);
     38      else OnRunnerMessage(message);
     39    }
    3540
    3641    public void Run() {
    3742      IPluginLoader loader = PluginLoaderFactory.Create();
    3843      loader.LoadPlugins(AssembliesToLoad);
    39       StartExecutor();
     44      Task t = Task.Run(Execute);
    4045      StartListener();
    41       executor.Join();
     46      t.Wait();
    4247    }
    4348
    4449    protected abstract void Execute();
    45     protected abstract void OnRunnerJob(RunnerJob runnerJob);
     50    protected abstract void OnRunnerMessage(RunnerMessage message);
    4651
    4752    #region Helper
    48     private void StartExecutor() {
    49       executor = new Thread(Execute);
    50       executor.IsBackground = false;
    51       executor.Start();
    52     }
    5353
    5454    private void StartListener() {
    5555      listener = new Thread(() => {
    5656        Stream stdin = Console.OpenStandardInput();
    57         while (executor.IsAlive)
    58           OnRunnerJob((RunnerJob)formatter.Deserialize(stdin));
     57        DateTime lastMessageRecieved = DateTime.MinValue;
     58        while (true) {
     59          RunnerMessage message = RunnerMessage.ReadMessageFromStream(stdin);
     60          if (DateTime.Compare(lastMessageRecieved, message.SendTime) < 0) {
     61            OnRunnerMessage(message);
     62            lastMessageRecieved = message.SendTime;
     63          }
     64        }
    5965      });
    6066      listener.IsBackground = true;

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/RunnerHost.cs

    r16993 r17013  
    44using System.Threading;
    55using System.Threading.Tasks;
     6using HEAL.Attic;
    67
    78namespace HeuristicLab.PluginInfrastructure {
    … …  
    1415    Starting,
    1516    Running,
    16     Paused,
    17     Stopping,
    1817    Stopped
    1918  }
    … …  
    2726    #region Vars
    2827    protected Process process;
     28    private ProtoBufSerializer serializer = new ProtoBufSerializer();
    2929    #endregion
    3030
    3131    #region Properties
     32    /// <summary>
     33    /// Set this to true, if console output should be disabled.
     34    /// </summary>
     35    public bool QuietMode { get; set; }
     36
     37
    3238    /// <summary>
    3339    /// The programm, which should be used. For example 'docker'.
    … …  
    4147    protected string Password { get; private set; }
    4248    protected string Domain { get; private set; }
     49    protected IRunner Runner { get; set; }
    4350
    44     /// <summary>
    45     /// The runner state.
    46     /// </summary>
    4751    public RunnerState State { get; protected set; } = RunnerState.Created;
    4852    #endregion
    4953
    5054    #region Constructors
     55
    5156    protected RunnerHost(string program, string startArgument, string userName, string password, string domain) {
    5257      Program = program;
    … …  
    6368        throw new InvalidOperationException("Runner must be in state 'Created'.");
    6469      State = RunnerState.Starting;
     70      Runner = runner;
    6571      process = new Process {
    6672        StartInfo = new ProcessStartInfo {
    … …  
    7278          RedirectStandardError = true,
    7379          CreateNoWindow = false,
    74           UserName = string.IsNullOrEmpty(UserName) ? null : UserName,
     80          UserName = string.IsNullOrEmpty(UserName) ? null : UserName, // TODO: accounts testen: https://docs.microsoft.com/en-us/windows/security/identity-protection/access-control/local-accounts#sec-localsystem
    7581          PasswordInClearText = string.IsNullOrEmpty(Password) ? null : Password,
    7682          Domain = string.IsNullOrEmpty(Domain) ? null : Domain,
    … …  
    8086      };
    8187      process.Start();
     88
    8289      // registers a task for cancellation, prevents the use of polling (while-loop)
    8390      var task = RegisterCancellation(token.HasValue ? token.Value : CancellationToken.None);
    8491
    85       // write config to standardinput, runner listens on this and deserializes the config
    86       Runner.Serialize(runner, process.StandardInput.BaseStream);
     92      // set runnerhost in runner
     93      Runner r = Runner as Runner;
     94      if (r != null) r.Host = this;
    8795
    8896      process.BeginOutputReadLine();
    8997      process.BeginErrorReadLine();
    9098
    91       if (!runner.QuietMode) {
     99      if (!QuietMode) {
    92100        process.OutputDataReceived += (s, e) => Console.WriteLine(e.Data);
    93101        process.ErrorDataReceived += (s, e) => Console.WriteLine(e.Data);
    94102      }
     103
     104      // write config to standardinput, runner listens on this and deserializes the config
     105      SendMessage(new TransportRunnerMessage(Runner));
     106
    95107      State = RunnerState.Running;
    96108      if (await task) State = RunnerState.Cancelled;
    97109      else State = RunnerState.Stopped;
    98 
    99110    }
    100111
    101     /// <summary>
    102     /// Pauses the runners. The childprocess gets notified with the help of the standard input and a serialized RunnerJob.
    103     /// To pause the runner, the state must be "Running".
    104     /// </summary>
    105     public void Pause() {
     112
     113    public void Send(RunnerMessage runnerMessage) {
    106114      if (State != RunnerState.Running) throw new InvalidOperationException("Runner must be in state 'Running'!");
    107       Runner.Pause(process.StandardInput.BaseStream);
    108       State = RunnerState.Paused;
     115      SendMessage(runnerMessage);
    109116    }
    110117
    111     /// <summary>
    112     /// Resumes a paused runner. The childprocess gets notified with the help of the standard input and a serialized RunnerJob.
    113     /// To resume the runner, the state must be "Paused".
    114     /// </summary>
    115     public void Resume() {
    116       if (State != RunnerState.Paused) throw new InvalidOperationException("Runner must be in state 'Paused'!");
    117       Runner.Resume(process.StandardInput.BaseStream);
    118       State = RunnerState.Running;
     118    // because we need to transfer the runner with a TransportRunnerMessage in the starting state and the
     119    // original send method should not be available until running state
     120    protected virtual void SendMessage(RunnerMessage runnerMessage) {
     121      runnerMessage.SendTime = DateTime.Now;
     122      byte[] bytes = serializer.Serialize(runnerMessage);
     123      byte[] size = BitConverter.GetBytes(bytes.Length);
     124      process.StandardInput.BaseStream.Write(size, 0, size.Length);
     125      process.StandardInput.BaseStream.Flush();
     126      process.StandardInput.BaseStream.Write(bytes, 0, bytes.Length);
     127      process.StandardInput.BaseStream.Flush();
    119128    }
    120129
    … …  
    125134    /// When the child process gets finished without requested cancellation, the linked token gets cancelled and a result set.
    126135    /// </summary>
    127     private Task<bool> RegisterCancellation(CancellationToken token) {
     136    protected Task<bool> RegisterCancellation(CancellationToken token) {
    128137      if (process != null && State == RunnerState.Starting) {
    129138        var cts = CancellationTokenSource.CreateLinkedTokenSource(token);
    … …  
    133142        cts.Token.Register(() => {
    134143          if (!process.HasExited) {
    135             Runner.Cancel(process.StandardInput.BaseStream);
     144            Runner.Cancel();
    136145            process.WaitForExit();
    137146          }

  • branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Utils/UniPath.cs

    r16984 r17013  
    1 using System;
    2 using System.Collections.Generic;
    3 using System.IO;
     1using System.IO;
    42using System.Runtime.InteropServices;
    5 using System.Text;
    63using System.Text.RegularExpressions;
     4using HEAL.Attic;
    75
    86namespace HeuristicLab.PluginInfrastructure {
    9   [Serializable]
     7  [StorableType("8911C8DC-FDA0-4CF6-A0CD-C67E72094D62")]
    108  public class UniPath {
    119
    1210    #region Vars
     11    [Storable]
    1312    private string fullPath = "";
    1413    #endregion
    1514
    1615    #region Constructors
     16    [StorableConstructor]
     17    private UniPath(StorableConstructorFlag _) { }
    1718    public UniPath(string path) {
    1819      fullPath = Path.GetFullPath(path);
    1920    }
    2021    #endregion
    21    
     22
    2223    private bool IsWindowsAbsolutePath(string path) => Regex.IsMatch(path, @"^[A-Z]:");
    23    
     24
    2425    private string Rebuild(char split, string startStr, string seperator) {
    2526      string[] splits = fullPath.Split(split);
    2627      string tmp = startStr;
    2728      int i = 1;
    28       while (i < splits.Length - 1) {
     29      while (i < (splits.Length - 1)) {
    2930        tmp += splits[i] + seperator;
    3031        ++i;
Note: See TracChangeset for help on using the changeset viewer.