source: branches/HeuristicLab.Hive-3.4/sources/HeuristicLab.Clients.Hive.Slave/3.4/PluginCache.cs @ 6101

#region License Information
/* HeuristicLab
 * Copyright (C) 2002-2010 Heuristic and Evolutionary Algorithms Laboratory (HEAL)
 *
 * This file is part of HeuristicLab.
 *
 * HeuristicLab is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * HeuristicLab is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with HeuristicLab. If not, see <http://www.gnu.org/licenses/>.
 */
#endregion

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.Serialization;
using System.Runtime.Serialization.Formatters.Binary;
using System.Threading;

namespace HeuristicLab.Clients.Hive.SlaveCore {
  public class PluginCache {
    private static object locker = new object();
    private const string lastUsedFileName = "lastUsed.dat";

    //maximum number of days after which a plugin gets deleted if not used
    private const int maxAge = 5;

    private static PluginCache instance = null;

    public string PluginCacheDir { get; set; }
    public string PluginTempBaseDir { get; set; }

    private List<Guid> cachedPluginsGuids = new List<Guid>();

    public static PluginCache Instance {
      get {
        if (instance == null)
          instance = new PluginCache();
        return instance;
      }
    }

    public PluginCache() {
      PluginCacheDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "PluginCache");
      PluginTempBaseDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "PluginTemp");

      DoUpdateRun();
    }

    private string GetFilenameFromPath(string path) {
      string[] dirParts = path.Split(Path.DirectorySeparatorChar);
      if (dirParts.Length > 0) {
        string fileGuid = dirParts[dirParts.Length - 1];
        return fileGuid;
      } else
        return "";
    }

    private void DoUpdateRun() {
      SafelyCreateDirectory(PluginCacheDir);
      foreach (string dir in Directory.EnumerateDirectories(PluginCacheDir)) {
        cachedPluginsGuids.Add(Guid.Parse(GetFilenameFromPath(dir))); // Todo: cleaner solution to getFilenameFromPath
      }
    }

    [MethodImpl(MethodImplOptions.Synchronized)]
    public void CopyPluginsForJob(List<Plugin> requests, Guid jobId, out string configFileName) {
      lock (locker) {
        configFileName = string.Empty;
        String targetDir = Path.Combine(PluginTempBaseDir, jobId.ToString());

        RecreateDirectory(targetDir);

        foreach (Plugin requestedPlugin in requests) {
          var filePaths = GetPluginFilePaths(requestedPlugin.Id);
          foreach (string filePath in filePaths) {
            File.Copy(filePath, Path.Combine(targetDir, Path.GetFileName(filePath)));
          }

          if (requestedPlugin.Name == "Configuration") {
            configFileName = Path.Combine(targetDir, Path.GetFileName(filePaths.SingleOrDefault())); // configuration plugin consists only of 1 file (usually the "HeuristicLab X.X.exe.config")
          }
        }

        // copy files from PluginInfrastructure (which are not declared in any plugins)
        string baseDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
        CopyFile(baseDir, targetDir, "HeuristicLab.PluginInfrastructure-3.3.dll");
        CopyFile(baseDir, targetDir, "ICSharpCode.SharpZipLib.dll");
        CopyFile(baseDir, targetDir, "ICSharpCode.SharpZipLib License.txt");

        // copy slave plugins, otherwise its not possible to register the UnhandledException handler to the appdomain        
        CopyFile(baseDir, targetDir, "HeuristicLab.Clients.Hive.SlaveCore-3.4.dll");
        CopyFile(baseDir, targetDir, "HeuristicLab.Clients.Hive-3.4.dll");
        //CopyFile(baseDir, targetDir, "HeuristicLab.Services.Hive.Common-3.4.dll");
        CopyFile(baseDir, targetDir, "HeuristicLab.Hive-3.4.dll");
        CopyFile(baseDir, targetDir, "HeuristicLab.Clients.Common-3.3.dll");
      }
    }

    private static DirectoryInfo RecreateDirectory(String targetDir) {
      var di = new DirectoryInfo(targetDir);
      if (di.Exists) Directory.Delete(targetDir, true);
      di.Refresh();
      while (di.Exists) {
        Thread.Sleep(50);
        di.Refresh();
      }
      return SafelyCreateDirectory(targetDir);
    }

    private static DirectoryInfo SafelyCreateDirectory(String targetDir) {
      var di = new DirectoryInfo(targetDir);
      if (!di.Exists) {
        di = Directory.CreateDirectory(targetDir);
        while (!di.Exists) {
          Thread.Sleep(50);
          di.Refresh();
        }
      }
      return di;
    }

    private void CopyFile(string baseDir, string targetDir, string fileName) {
      if (!File.Exists(Path.Combine(targetDir, fileName))) File.Copy(Path.Combine(baseDir, fileName), Path.Combine(targetDir, fileName));
    }

    [MethodImpl(MethodImplOptions.Synchronized)]
    internal void PreparePlugins(Job myJob, out string configFileName) {
      lock (locker) {
        SlaveClientCom.Instance.ClientCom.LogMessage("Fetching plugins for job");

        List<Plugin> requiredPlugins = new List<Plugin>();
        foreach (Guid pluginId in myJob.PluginsNeededIds) {
          //TODO: rewrite, GetPlugin(..) should be the last thing to do after checking the local plugins
          Plugin plugin = WcfService.Instance.GetPlugin(pluginId);
          if (plugin != null)
            requiredPlugins.Add(plugin);
        }

        List<Plugin> localPlugins = new List<Plugin>();
        List<Plugin> missingPlugins = new List<Plugin>();
        List<Guid> missingGuids = new List<Guid>();
        bool found = false;

        foreach (Plugin info in requiredPlugins) {
          foreach (Guid cachedPlugin in cachedPluginsGuids) {
            if (info.Id == cachedPlugin) {
              localPlugins.Add(new Plugin() { Id = cachedPlugin, Name = info.Name, Version = info.Version });
              found = true;

              break;
            }
          }
          if (!found) {
            SlaveClientCom.Instance.ClientCom.LogMessage("Plugin NOT found " + info.Name + ", " + info.Version);
            missingPlugins.Add(info);
            missingGuids.Add(info.Id);
          }
          found = false;
        }

        SlaveClientCom.Instance.ClientCom.LogMessage("First run - Update the plugins in the cache");
        localPlugins.AddRange(missingPlugins);

        IEnumerable<PluginData> pluginDatas = WcfService.Instance.GetPluginDatas(missingGuids);

        if (pluginDatas != null) {
          foreach (PluginData pluginData in pluginDatas) {
            string pluginDir = Path.Combine(PluginCacheDir, pluginData.PluginId.ToString());

            //put all files belonging to a plugin in the same directory
            SafelyCreateDirectory(pluginDir);
            File.WriteAllBytes(Path.Combine(pluginDir, Path.GetFileName(pluginData.FileName)), pluginData.Data);
          }

          DoUpdateRun();
          CopyPluginsForJob(requiredPlugins, myJob.Id, out configFileName);
        } else {
          configFileName = "";
        }
      }
    }

    /// <summary>
    /// Returns a list of files which belong to a plugin from the plugincache
    /// </summary>
    private IEnumerable<string> GetPluginFilePaths(Guid pluginId) {
      string pluginPath = Path.Combine(PluginCacheDir, pluginId.ToString());

      if (Directory.Exists(pluginPath)) {
        WriteDateLastUsed(pluginPath);
        foreach (string filePath in Directory.GetFiles(pluginPath)) {
          string fn = Path.GetFileName(filePath);
          if (fn != lastUsedFileName)
            yield return filePath;
        }
      }
    }

    /// <summary>
    /// creates a file in path with the current date;
    /// this can later be used to find plugins which are outdated
    /// </summary>    
    private void WriteDateLastUsed(string path) {
      FileStream fs = new FileStream(Path.Combine(path, lastUsedFileName), FileMode.Create);
      BinaryFormatter formatter = new BinaryFormatter();

      try {
        formatter.Serialize(fs, DateTime.Now);
      }
      catch (SerializationException) {
        //rethrow...
        throw;
      }
      finally {
        fs.Close();
      }
    }

    /// <summary>
    /// Checks the PluginTemp directory for orphaned directories and deletes them.
    /// This should be only called if no jobs are currently running.
    /// </summary>    
    public void CleanPluginTemp() {
      if (Directory.Exists(PluginTempBaseDir)) {
        foreach (string dir in Directory.EnumerateDirectories(PluginTempBaseDir)) {
          try {
            SlaveClientCom.Instance.ClientCom.LogMessage("Deleting orphaned directory " + dir);
            Directory.Delete(dir, true);
          }
          catch (Exception ex) {
            SlaveClientCom.Instance.ClientCom.LogMessage("Error cleaning up PluginTemp directory " + dir + ": " + ex.ToString());
          }
        }
      }
    }

    /// <summary>
    /// checks the pluginCacheDirectory and deletes plugin folders which are not used anymore
    /// </summary>    
    private void CleanPluginCache() {
      FileStream fs = null;
      DateTime luDate;
      bool changed = false;

      if (Directory.Exists(PluginCacheDir)) {
        foreach (string dir in Directory.EnumerateDirectories(PluginCacheDir)) {
          try {
            fs = new FileStream(Path.Combine(dir, lastUsedFileName), FileMode.Open);
            BinaryFormatter formatter = new BinaryFormatter();
            luDate = (DateTime)formatter.Deserialize(fs);
            fs.Close();

            if (luDate.AddDays(maxAge) < DateTime.Now) {
              Directory.Delete(dir, true);
              changed = true;
            }
          }
          catch (SerializationException) {
            fs.Close();
            throw;
          }
          catch (System.IO.FileNotFoundException) {
            //nerver used
            Directory.Delete(dir, true);
            changed = true;
          }
          catch (Exception) {
            throw;
          }
        }
      }
      if (changed)
        DoUpdateRun();
    }


    internal void DeletePluginsForJob(Guid id) {
      try {
        SlaveClientCom.Instance.ClientCom.LogMessage("unloading...");
        int tries = 5;
        while (tries > 0) {
          try {
            Directory.Delete(Path.Combine(PluginTempBaseDir, id.ToString()), true);
            tries = 0;
          }
          catch (Exception e) {
            Thread.Sleep(1000);
            tries--;
            if (tries == 0) throw;// TODO: don't know what do do
          }
        }
      }
      catch (Exception ex) {
        SlaveClientCom.Instance.ClientCom.LogMessage("failed while unloading " + id + " with exception " + ex);
      }
      CleanPluginCache();
    }
  }
}