Changeset 17013 for branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation

Timestamp:

06/17/19 10:40:41 (5 years ago)

Author:

dpiringe

Message:

#2924:

• some changes in CLIApplication.cs to reduce unnecessary allocation of string objects
• renamed AppTest to ConsoleOptimizer and fixed race condition
• replaced enum RunnerJob with class RunnerMessage for more control of saved data
• changed usage of BinaryFormatter with HEAL.Attic, following types are now Storable:
  • ConsoleOptimizer
  • InspectApplication
  • ApplicationBase
  • ApplicationRunner
  • AssemblyInfo
  • Runner
  • UniPath
  • RunnerMessage
• switched QuietMode from ApplicationRunner to IRunner
• DockerRunnerHost can now automatically build docker images for linux and windows containers (also identifies which container type is active) -> removes the condition to have the image preinstalled
  • to achieve this, there are two new folders DockerLinuxBuild and DockerWindowsBuild included in build output, which include Dockerfiles to build images for linux and windows container
• added NuGet package System.CodeDom to project HeuristicLab.Scripting-3.3
• added method Send(RunnerMessage) to IRunnerHost and transferred methods Pause and Resume to IRunner
• added internal reference of RunnerHost in Runner
• added abstract method SendMessage(RunnerMessage) in RunnerHost which gets called from method Send(RunnerMessage)
• because of a Google.Protobuf "bug", RunnerMessages cannot get serialized/deserialized directly on stream -> workaround with a byte array, which gets written and read
  • additionally, the length of the array gets sent first (length as integer -> 4 bytes)
  • static method in RunnerMessage to read a message from a stream
  • the method SendMessage(RunnerMessage) in RunnerHost implements this functionality

Location:

branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation

Files:

• ApplicationRunner.cs (modified) (4 diffs)
• AssemblyInfo.cs (modified) (1 diff)
• DockerRunnerHost.cs (modified) (3 diffs)
• Runner.cs (modified) (1 diff)
• RunnerHost.cs (modified) (9 diffs)
• RunnerJob.cs (deleted)
• RunnerMessage.cs (added)

branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/ApplicationRunner.cs

@@ -1 @@
-using System;
-using System.IO;
+using HEAL.Attic;
 
 namespace HeuristicLab.PluginInfrastructure {
-  [Serializable]
+  [StorableType("612F98AF-E254-4C5E-BD41-75B4F1D9B96D")]
   public class ApplicationRunner : Runner {
     /// <summary>
     /// Arguments for the StartApplication.
     /// </summary>
+    [Storable]
     public ICommandLineArgument[] Args { get; set; }
 
@@ -15 +15 @@
     public IApplication StartApplication {
       get {
-        if (application == null) {
-          using (var memStream = new MemoryStream(serializedStartApplication)) {
-            application = (IApplication)formatter.Deserialize(memStream);
-          }
+        lock (locker) {
+          if (application == null)
+            application = (IApplication)new ProtoBufSerializer().Deserialize(serializedStartApplication);
+          return application;
         }
-        return application;
       }
       set {
-        using (var ms = new MemoryStream()) {
-          formatter.Serialize(ms, value);
-          serializedStartApplication = ms.ToArray();
+        lock (locker) {
+          serializedStartApplication = new ProtoBufSerializer().Serialize(value);
+          application = value;
         }
       }
@@ -31 +30 @@
     // Encapsulated application is necessary, because it is not possible to 
     // instantly deserialize the application, before all assemblies are loaded.
+    [Storable]
     private byte[] serializedStartApplication = new byte[0];
+
     // cache application to prevent new instances every get call of StartApplication
     private IApplication application;
+    private object locker = new object();
 
     protected override void Execute() {
@@ -39 +41 @@
     }
 
-    protected override void OnRunnerJob(RunnerJob runnerJob) {
-      switch (runnerJob) {
-        case RunnerJob.Cancel: StartApplication.OnCancel(); break;
-        case RunnerJob.Pause: StartApplication.OnPause(); break;
-        case RunnerJob.Resume: StartApplication.OnResume(); break;
-      }
+    protected override void OnRunnerMessage(RunnerMessage message) {
+      if (message is PauseRunnerMessage)
+        StartApplication.OnPause();
+      else if (message is ResumeRunnerMessage)
+        StartApplication.OnResume();
+      else if (message is CancelRunnerMessage)
+        StartApplication.OnCancel();
     }
   }

branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/AssemblyInfo.cs

@@ -1 @@
-using System;
+using HEAL.Attic;
 
 namespace HeuristicLab.PluginInfrastructure {
-  [Serializable]
+  [StorableType(StorableMemberSelection.AllProperties, "D1DC9C3E-16B7-464E-A377-35FA614195AD")]
   public class AssemblyInfo {
     /// <summary>

branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/DockerRunnerHost.cs

@@ -1 +1 @@
 using System.Diagnostics;
+using System.IO;
+using System.Threading.Tasks;
 using HeuristicLab.PluginInfrastructure.Exceptions;
 
@@ -8 +10 @@
   public class DockerRunnerHost : RunnerHost {
     #region Constants
-    private const string Docker = "docker";
-    private const string ContainerStartup = "container run -i --rm ";
-    private const string Mounting = @"--mount type=bind,source=/c/Users/,target=/Users ";
-    private const string Image = "heuristiclab33:latest";
-    private const string DockerExceptionMessage = "Docker is not running or image '" + Image + "' does not exists!";
+    private readonly static string Docker = "docker";
+    private readonly static string ContainerStartup = "container run -i --rm ";
+    private readonly static string MountingLinux = @"--mount type=bind,source=/c/Users/,target=/Users ";
+    private readonly static string MountingWindows = @"--mount type=bind,source=C:\Users\,target=C:\Users\ ";
+    private readonly static string LinuxImage = "." + Path.DirectorySeparatorChar + "DockerLinuxBuild" + Path.DirectorySeparatorChar + "Dockerfile";
+    private readonly static string WindowsImage = "." + Path.DirectorySeparatorChar + "DockerWindowsBuild" + Path.DirectorySeparatorChar + "Dockerfile";
+    private readonly static string Image = "heuristiclab33:latest";
+    private readonly static string DockerExceptionMessage = "Docker is not running!";
     #endregion
 
     #region Constructors
-    public DockerRunnerHost(bool doDockerAvailableCheck = true)
-      : base(Docker, ContainerStartup + Mounting + Image, null, null, null) {
-      if (doDockerAvailableCheck && !IsDockerAvailable())
-        throw new DockerException(DockerExceptionMessage);
+    public DockerRunnerHost()
+      : base(Docker, ContainerStartup + GetTargetOSMounting() + Image, null, null, null) {
     }
     #endregion
 
     #region Helper
-    private bool IsDockerAvailable() {
+
+    private static string GetTargetOSMounting() {
+      Task<bool> win = Task.Run(() => BuildImage(WindowsImage));
+      Task<bool> lin = Task.Run(() => BuildImage(LinuxImage));
+      if (win.Result) return MountingWindows;
+      else if (lin.Result) return MountingLinux;
+      else throw new DockerException(DockerExceptionMessage);
+    }
+
+    private static bool BuildImage(string pathToDockerfile) {
       var process = new Process {
         StartInfo = new ProcessStartInfo {
           FileName = Docker,
-          Arguments = "image inspect " + Image,
+          Arguments = "image build -t " + Image +
+            " -f " + Path.GetFullPath(pathToDockerfile) +
+            " .",
           UseShellExecute = false,
           RedirectStandardOutput = true,
@@ -37 +51 @@
         EnableRaisingEvents = false
       };
-
       process.Start();
       process.BeginOutputReadLine();

branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/Runner.cs

@@ -2 +2 @@
 using System.Collections.Generic;
 using System.IO;
-using System.Runtime.Serialization;
-using System.Runtime.Serialization.Formatters.Binary;
 using System.Threading;
+using System.Threading.Tasks;
+using HEAL.Attic;
 
 namespace HeuristicLab.PluginInfrastructure {
-  [Serializable]
+  [StorableType(StorableMemberSelection.MarkedOnly, "D93CBE04-9847-417A-AAB5-7FBCA6A32247")]
   public abstract class Runner : IRunner {
 
     #region Vars
-    [NonSerialized]
     private Thread listener;
-
-    [NonSerialized]
-    private Thread executor;
-
-    [NonSerialized]
-    protected static IFormatter formatter = new BinaryFormatter();
     #endregion
 
     #region Properties
-    public bool QuietMode { get; set; }
+    [Storable]
+    public IEnumerable<AssemblyInfo> AssembliesToLoad { get; set; }
 
-    public IEnumerable<AssemblyInfo> AssembliesToLoad { get; set; }
+    internal RunnerHost Host { get; set; }
     #endregion
 
+    public void Pause() {
+      var message = new PauseRunnerMessage();
+      if (Host != null) Host.Send(message);
+      else OnRunnerMessage(message);
+    }
 
-    public static void Serialize(IRunner runner, Stream stream) => formatter.Serialize(stream, runner);
-    public static IRunner Deserialize(Stream stream = null) => (IRunner)formatter.Deserialize(stream);
-    public static void Pause(Stream stream) => formatter.Serialize(stream, RunnerJob.Pause);
-    public static void Resume(Stream stream) => formatter.Serialize(stream, RunnerJob.Resume);
-    public static void Cancel(Stream stream) => formatter.Serialize(stream, RunnerJob.Cancel);
+    public void Resume() {
+      var message = new ResumeRunnerMessage();
+      if (Host != null) Host.Send(message);
+      else OnRunnerMessage(message);
+    }
+
+    public void Cancel() {
+      var message = new CancelRunnerMessage();
+      if (Host != null) Host.Send(message);
+      else OnRunnerMessage(message);
+    }
 
     public void Run() {
       IPluginLoader loader = PluginLoaderFactory.Create();
       loader.LoadPlugins(AssembliesToLoad);
-      StartExecutor();
+      Task t = Task.Run(Execute);
       StartListener();
-      executor.Join();
+      t.Wait();
     }
 
     protected abstract void Execute();
-    protected abstract void OnRunnerJob(RunnerJob runnerJob);
+    protected abstract void OnRunnerMessage(RunnerMessage message);
 
     #region Helper
-    private void StartExecutor() {
-      executor = new Thread(Execute);
-      executor.IsBackground = false;
-      executor.Start();
-    }
 
     private void StartListener() {
       listener = new Thread(() => {
         Stream stdin = Console.OpenStandardInput();
-        while (executor.IsAlive)
-          OnRunnerJob((RunnerJob)formatter.Deserialize(stdin));
+        DateTime lastMessageRecieved = DateTime.MinValue;
+        while (true) {
+          RunnerMessage message = RunnerMessage.ReadMessageFromStream(stdin);
+          if (DateTime.Compare(lastMessageRecieved, message.SendTime) < 0) {
+            OnRunnerMessage(message);
+            lastMessageRecieved = message.SendTime;
+          }
+        }
       });
       listener.IsBackground = true;

branches/2924_DotNetCoreMigration/HeuristicLab.PluginInfrastructure/3.3/Isolation/RunnerHost.cs

@@ -4 +4 @@
 using System.Threading;
 using System.Threading.Tasks;
+using HEAL.Attic;
 
 namespace HeuristicLab.PluginInfrastructure {
@@ -14 +15 @@
     Starting,
     Running,
-    Paused,
-    Stopping,
     Stopped
   }
@@ -27 +26 @@
     #region Vars
     protected Process process;
+    private ProtoBufSerializer serializer = new ProtoBufSerializer();
     #endregion
 
     #region Properties
+    /// <summary>
+    /// Set this to true, if console output should be disabled.
+    /// </summary>
+    public bool QuietMode { get; set; }
+
+
     /// <summary>
     /// The programm, which should be used. For example 'docker'.
@@ -41 +47 @@
     protected string Password { get; private set; }
     protected string Domain { get; private set; }
+    protected IRunner Runner { get; set; }
 
-    /// <summary>
-    /// The runner state.
-    /// </summary>
     public RunnerState State { get; protected set; } = RunnerState.Created;
     #endregion
 
     #region Constructors
+
     protected RunnerHost(string program, string startArgument, string userName, string password, string domain) {
       Program = program;
@@ -63 +68 @@
         throw new InvalidOperationException("Runner must be in state 'Created'.");
       State = RunnerState.Starting;
+      Runner = runner;
       process = new Process {
         StartInfo = new ProcessStartInfo {
@@ -72 +78 @@
           RedirectStandardError = true,
           CreateNoWindow = false,
-          UserName = string.IsNullOrEmpty(UserName) ? null : UserName,
+          UserName = string.IsNullOrEmpty(UserName) ? null : UserName, // TODO: accounts testen: https://docs.microsoft.com/en-us/windows/security/identity-protection/access-control/local-accounts#sec-localsystem
           PasswordInClearText = string.IsNullOrEmpty(Password) ? null : Password,
           Domain = string.IsNullOrEmpty(Domain) ? null : Domain,
@@ -80 +86 @@
       };
       process.Start();
+
       // registers a task for cancellation, prevents the use of polling (while-loop)
       var task = RegisterCancellation(token.HasValue ? token.Value : CancellationToken.None);
 
-      // write config to standardinput, runner listens on this and deserializes the config
-      Runner.Serialize(runner, process.StandardInput.BaseStream);
+      // set runnerhost in runner 
+      Runner r = Runner as Runner;
+      if (r != null) r.Host = this;
 
       process.BeginOutputReadLine();
       process.BeginErrorReadLine();
 
-      if (!runner.QuietMode) {
+      if (!QuietMode) {
         process.OutputDataReceived += (s, e) => Console.WriteLine(e.Data);
         process.ErrorDataReceived += (s, e) => Console.WriteLine(e.Data);
       }
+
+      // write config to standardinput, runner listens on this and deserializes the config
+      SendMessage(new TransportRunnerMessage(Runner));
+
       State = RunnerState.Running;
       if (await task) State = RunnerState.Cancelled;
       else State = RunnerState.Stopped;
-
     }
 
-    /// <summary>
-    /// Pauses the runners. The childprocess gets notified with the help of the standard input and a serialized RunnerJob.
-    /// To pause the runner, the state must be "Running".
-    /// </summary>
-    public void Pause() {
+
+    public void Send(RunnerMessage runnerMessage) {
       if (State != RunnerState.Running) throw new InvalidOperationException("Runner must be in state 'Running'!");
-      Runner.Pause(process.StandardInput.BaseStream);
-      State = RunnerState.Paused;
+      SendMessage(runnerMessage);
     }
 
-    /// <summary>
-    /// Resumes a paused runner. The childprocess gets notified with the help of the standard input and a serialized RunnerJob.
-    /// To resume the runner, the state must be "Paused".
-    /// </summary>
-    public void Resume() {
-      if (State != RunnerState.Paused) throw new InvalidOperationException("Runner must be in state 'Paused'!");
-      Runner.Resume(process.StandardInput.BaseStream);
-      State = RunnerState.Running;
+    // because we need to transfer the runner with a TransportRunnerMessage in the starting state and the 
+    // original send method should not be available until running state
+    protected virtual void SendMessage(RunnerMessage runnerMessage) {
+      runnerMessage.SendTime = DateTime.Now;
+      byte[] bytes = serializer.Serialize(runnerMessage);
+      byte[] size = BitConverter.GetBytes(bytes.Length);
+      process.StandardInput.BaseStream.Write(size, 0, size.Length);
+      process.StandardInput.BaseStream.Flush();
+      process.StandardInput.BaseStream.Write(bytes, 0, bytes.Length);
+      process.StandardInput.BaseStream.Flush();
     }
 
@@ -125 +134 @@
     /// When the child process gets finished without requested cancellation, the linked token gets cancelled and a result set. 
     /// </summary>
-    private Task<bool> RegisterCancellation(CancellationToken token) {
+    protected Task<bool> RegisterCancellation(CancellationToken token) {
       if (process != null && State == RunnerState.Starting) {
         var cts = CancellationTokenSource.CreateLinkedTokenSource(token);
@@ -133 +142 @@
         cts.Token.Register(() => {
           if (!process.HasExited) {
-            Runner.Cancel(process.StandardInput.BaseStream);
+            Runner.Cancel();
             process.WaitForExit();
           }