using System; using System.Collections.Generic; using System.Linq; using System.Text; using HeuristicLab.Services.Optimization.ControllerService.Interfaces; using HeuristicLab.Optimization; using HeuristicLab.Algorithms.GeneticAlgorithm; using HeuristicLab.Problems.TravelingSalesman; using HeuristicLab; using System.Reflection; using HeuristicLab.Services.Optimization.ControllerService.Model; using HeuristicLab.Core; using System.Collections; using HeuristicLab.Clients.Hive; using System.Threading; using HeuristicLab.Data; using System.IO; using Microsoft.WindowsAzure; using Microsoft.WindowsAzure.StorageClient; using HeuristicLab.Services.Optimization.ControllerService.Azure; using System.Data; using HeuristicLab.Services.Optimization.ControllerService.General; namespace HeuristicLab.Services.Optimization.ControllerService { public class ScenarioEntity : TableServiceEntity { public ScenarioEntity() { } public ScenarioEntity(string scenarioName, string scenario, string mapper) { PartitionKey = "ScenarioPartition"; RowKey = scenarioName; Scenario = scenario; Mapper = mapper; } public string Scenario { get; set; } public string Mapper { get; set; } } public class HiveScenarioManager : IScenarioManager { private static IScenarioMapper tspMapper; private static object lockable; private Dictionary mappers = new Dictionary(); private IDataAccessLayer dal = DataAccessLayerProvider.GetLayer(); public static readonly string SCENARIO_TABLE = "Scenario"; static HiveScenarioManager() { lockable = new object(); } public string DispatchScenario(Model.User user, Model.OptimizationScenario scenario, JobExecutionDetails details) { // Experiment experiment = new Experiment(); // var problem = new TravelingSalesmanProblem(); // var algo = new GeneticAlgorithm(); // algo.Problem = problem; IScenarioMapper mapper = GetMapper(scenario.Id); IAlgorithm algo; mapper.MapScenario(scenario, out algo); /*if (details.Repititions > 1) { BatchRun br = new BatchRun(); //br.Name = details.JobTitle; br.Optimizer = algo; br.Repetitions = details.Repititions; return SendExperimentToHive(user, br, details); } else { }*/ return SendExperimentToHive(user, algo, details); } private sealed class StackEntry { public HeuristicLab.Optimization.Experiment Parent { get; set; } public IList Children { get; set; } } public bool DispatchExperiment(User user, Model.Experiment exp, JobExecutionDetails details) { // TODO: Determine how to build a tree of IAlgorithm // For now the experiment will be flatened for execution HeuristicLab.Optimization.Experiment hiveExperiment = new HeuristicLab.Optimization.Experiment(exp.Name); var stack = new Stack(); var children = new List(); foreach (var child in exp.Algorithm) { children.Add(child); } stack.Push(new StackEntry() { Parent = hiveExperiment, Children = children }); while (stack.Count > 0) { var entry = stack.Pop(); // handle current entry // TODO: Store scenario name in entry.Child.Mapper when saving foreach (var child in entry.Children) { // This is a template experiment; if (child.Name == null) { var parent = new HeuristicLab.Optimization.Experiment(child.Name); entry.Parent.Optimizers.Add(parent); stack.Push(new StackEntry() { Parent = parent, Children = child.ChildAlgorithms }); } // This entity is mapable else { IScenarioMapper mapper = GetMapper(child.Name); if (mapper == null) { // TODO: We should really be able to difference Experiment/Algorithm types; this is a workaround var parent = new HeuristicLab.Optimization.Experiment(child.Name); entry.Parent.Optimizers.Add(parent); stack.Push(new StackEntry() { Parent = parent, Children = child.ChildAlgorithms }); } else { var optScen = new OptimizationScenario() { Id = child.Name }; optScen.Algorithm.Add(child); IAlgorithm algo; mapper.MapScenario(optScen, out algo); entry.Parent.Optimizers.Add(algo); } } } } details.JobTitle = exp.Name; return SendExperimentToHive(user, hiveExperiment, details) != null; } static public string AssemblyDirectory { get { string codeBase = Assembly.GetExecutingAssembly().CodeBase; UriBuilder uri = new UriBuilder(codeBase); string path = Uri.UnescapeDataString(uri.Path); return Path.GetDirectoryName(path); } } public static IScenarioMapper CompileMapper(string scenarioMapper) { // http://stackoverflow.com/questions/3188882/compile-and-run-dynamic-code-without-generating-exe using (var csCodeProvider = new Microsoft.CSharp.CSharpCodeProvider()) { var cp = new System.CodeDom.Compiler.CompilerParameters() { GenerateInMemory = true }; var referencedPaths = Directory.GetFiles(AssemblyDirectory, "*.dll"); foreach (var loadedAssembly in referencedPaths) { cp.ReferencedAssemblies.Add(loadedAssembly); } cp.ReferencedAssemblies.Add("System.dll"); cp.ReferencedAssemblies.Add("System.Core.dll"); cp.ReferencedAssemblies.Add("System.Data.dll"); cp.ReferencedAssemblies.Add("System.Xml.dll"); cp.ReferencedAssemblies.Add("System.Xml.Linq.dll"); var res = csCodeProvider.CompileAssemblyFromSource( cp, scenarioMapper ); foreach (var error in res.Errors) { Console.WriteLine(error); } var firstMapper = res.CompiledAssembly.GetTypes().Where(p => typeof(IScenarioMapper).IsAssignableFrom(p)).FirstOrDefault(); return Activator.CreateInstance(firstMapper) as IScenarioMapper; } } private IScenarioMapper GetMapper(string scenarioId) { var id = scenarioId; if (!mappers.ContainsKey(id)) { lock (lockable) { if (mappers.ContainsKey(id)) return mappers[id]; var mapperString = GetMapperFromBlobStore(id); if (mapperString == null) return null; var mapper = CompileMapper(mapperString); mappers[id] = mapper; return mapper; } // lock } // if return null; } private void MapExperiment(Model.OptimizationScenario scenario, out IAlgorithm algorithm) { IScenarioMapper mapper = GetMapper(scenario.Id); mapper.MapScenario(scenario, out algorithm); } private HiveServiceLocator ConfigureHive(Model.User user) { var serviceLocator = new HiveServiceLocator(); serviceLocator.Username = user.Username; serviceLocator.Password = user.Password; serviceLocator.EndpointConfigurationName = Configuration.HiveEndpointName; return serviceLocator; } private string SendExperimentToHive(Model.User user, IOptimizer exp, JobExecutionDetails details) { var job = new RefreshableJob(); job.IsAllowedPrivileged = true; job.Job.Name = details.JobTitle; job.Job.ResourceNames = details.Group; job.RefreshAutomatically = false; if (details.Repititions > 1) { BatchRun br = new BatchRun(); br.Optimizer = exp; br.Repetitions = details.Repititions; exp = br; } if (exp.ExecutionState != ExecutionState.Prepared) { exp.Prepare(); } job.HiveTasks.Add(new OptimizerHiveTask(exp)); var service = ConfigureHive(user); // TODO: Fix HiveClient class to be not dependent on singleton HiveServiceLocator!!! HiveServiceLocator.Instance.Username = user.Username; HiveServiceLocator.Instance.Password = user.Password; HiveServiceLocator.Instance.EndpointConfigurationName = Configuration.HiveEndpointName; HiveClient.StartJob((ex) => { Console.WriteLine(ex.StackTrace); }, job, new CancellationToken()); job.StopResultPolling(); return job.Id != null ? job.Id.ToString() : null; } public IList GetJobs(User user) { var serviceLocator = ConfigureHive(user); var jobsLoaded = serviceLocator.CallHiveService>(s => s.GetJobs()); IList jobs = new List(); foreach (var job in jobsLoaded) { jobs.Add(ConvertJob(user, job)); } return jobs; } private Model.Job ConvertJob(User user, HeuristicLab.Clients.Hive.Job job) { var waitingJobs = job.JobCount - job.CalculatingCount - job.FinishedCount; Model.JobState? state = null; if (job.CalculatingCount > 0) state = JobState.Calculating; else if (job.JobCount > 0 && job.JobCount == job.FinishedCount) state = JobState.Finished; else state = JobState.Waiting; return new Model.Job() { Id = job.Id.ToString(), Name = job.Name, Resource = job.ResourceNames, State = state.Value, DateCreated = job.DateCreated }; } public Model.Job GetJob(User user, string id) { var serviceLocator = ConfigureHive(user); var guid = Guid.Parse(id); return ConvertJob(user, serviceLocator.CallHiveService(s => s.GetJob(guid))); } public bool DeleteJob(User user, string id) { var serviceLocator = ConfigureHive(user); var guid = Guid.Parse(id); serviceLocator.CallHiveService(s => s.DeleteJob(guid)); return true; } public IList GetJobResults(User user, string id) { var serviceLocator = ConfigureHive(user); var guid = Guid.Parse(id); var jobTasks = serviceLocator.CallHiveService>(s => s.GetLightweightJobTasks(guid)); IList taskIds = new List(); foreach (var task in jobTasks) { taskIds.Add(task.Id); } // TODO: Fix problems with the HiveServiceLocater singleton!!! HiveServiceLocator.Instance.Username = user.Username; HiveServiceLocator.Instance.Password = user.Password; HiveServiceLocator.Instance.EndpointConfigurationName = Configuration.HiveEndpointName; TaskDownloader downloader = new TaskDownloader(taskIds); downloader.StartAsync(); while (!downloader.IsFinished) { Thread.Sleep(500); if (downloader.IsFaulted) { throw downloader.Exception; } } IDictionary hiveTasks = downloader.Results; IList runs = new List(); foreach (var keyTask in hiveTasks.Keys) { var oht = hiveTasks[keyTask] as OptimizerHiveTask; if (oht != null) { foreach (var run in oht.ItemTask.Item.Runs) { Model.Run taskRun = new Model.Run(); taskRun.Id = taskRun.Name = run.Name; IList resultValues = new List(); foreach (var key in run.Results.Keys) { var value = run.Results[key]; Parameter result = MapHiveDataType(key, value); resultValues.Add(result); } taskRun.Results = resultValues; runs.Add(taskRun); } } } return runs; } //TODO: We might need images / +++ private Parameter MapHiveDataType(string name, IItem item) { Parameter result = new Parameter(); result.Type = ParameterType.String; //TODO: How shall we handle dll specific datatypes? //if (item is PathTSPTour) { // var tour = item as PathTSPTour; //} //else if (item is IStringConvertibleValue) { var value = (item as IStringConvertibleValue).GetValue(); result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value }; } else if (item is IStringConvertibleValueTuple) { var value1 = (item as IStringConvertibleValueTuple).Item1.GetValue(); var value2 = (item as IStringConvertibleValueTuple).Item2.GetValue(); result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = "{" + value1 + ", " + value2 + "}" }; } else if (item is DoubleArray) { var array = item as DoubleArray; double[] arrayValue = new double[array.Length]; for (int i = 0; i < arrayValue.Length; ++i) { arrayValue[i] = array[i]; } result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.DecimalVector() { Name = name, Value = arrayValue }; } else if (item is DoubleMatrix) { var matrix = item as DoubleMatrix; double[][] matrixValue = new double[matrix.Rows][]; for (int i = 0; i < matrixValue.Length; ++i) { matrixValue[i] = new double[matrix.Columns]; for (int j = 0; j < matrixValue[i].Length; ++j) { matrixValue[i][j] = matrix[i, j]; } } result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.DecimalMatrix() { Name = name, Value = matrixValue }; } else if (item is IStringConvertibleArray) { StringBuilder sb = new StringBuilder(); var array = item as IStringConvertibleArray; if (array.Length == 0) { sb.Append("[ ]"); } else { sb.Append("["); for (int i = 0; i < array.Length - 1; i++) { sb.Append(array.GetValue(i)).Append(", "); } sb.Append(array.GetValue(array.Length - 1)); sb.Append(" ]"); } var value = sb.ToString(); result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value }; } else if (item is HeuristicLab.Analysis.DataTable) { var table = item as HeuristicLab.Analysis.DataTable; string[] names = new string[table.Rows.Count]; double[][] results = new double[table.Rows.Count][]; for (int i = 0; i < table.Rows.Count; i++ ) { var columns = table.Rows.ToList()[i]; names[i] = columns.Name; results[i] = new double[columns.Values.Count]; for (int j=0; j < columns.Values.Count; j++) { results[i][j] = columns.Values[j]; } } result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.DecimalMatrix() { Name = name, Value = results, RowNames = names }; } else if (item is IStringConvertibleMatrix) { StringBuilder sb = new StringBuilder(); var matrix = item as IStringConvertibleMatrix; if (matrix.Rows == 0 || matrix.Columns == 0) { sb.Append("[ ]"); } else { sb.Append("[ "); for (int r = 0; r < matrix.Rows; r++) { sb.Append("( "); for (int c = 0; c < matrix.Columns - 1; c++) { matrix.GetValue(r, c); sb.Append(matrix.GetValue(r, c)).Append(", "); } sb.Append(matrix.GetValue(r, matrix.Columns - 1)).Append(r < matrix.Rows - 1 ? " ), " : " )"); } sb.Append(" ]"); } var value = sb.ToString(); result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = value }; } else { result.Value = new HeuristicLab.Services.Optimization.ControllerService.Model.StringValue() { Name = name, Value = "Cannot be displayed as string" }; } // TODO: Add workaround for TSP return result; } public bool AddScenario(User user, string scenarioName, string scenarioXml, string scenarioMapper) { // create scenario mapper var mapper = CompileMapper(scenarioMapper); if (mapper == null) return false; // insert into table & blob store var scenDao = dal.ScenarioDao; var blobDao = dal.BlobDao; Guid scenarioXmlGuid = Guid.NewGuid(); Guid scenarioMapperGuid = Guid.NewGuid(); string scenarioXmlId = scenarioName + "_" + scenarioXmlGuid.ToString(); string scenarioMapperId = scenarioName + "_" + scenarioMapperGuid.ToString(); if (!blobDao.Add(new StringEntry() { Key = scenarioXmlId, Text = scenarioXml })) return false; if (!blobDao.Add(new StringEntry() { Key = scenarioMapperId, Text = scenarioMapper })) return false; if (!scenDao.Add(new ScenarioEntity(scenarioName, scenarioXmlId, scenarioMapperId))) return false; // everything stored in the DB -> add mapper to dictionary if (!mappers.ContainsKey(scenarioName)) { lock (lockable) { if (!mappers.ContainsKey(scenarioName)) mappers[scenarioName] = mapper; } } return true; } private string GetMapperFromBlobStore(string scenarioName) { var scenarioDao = dal.ScenarioDao; var blobDao = dal.BlobDao; var entity = scenarioDao.FindByName(scenarioName); if (entity == null) return null; var mapper = blobDao.FindByKey(entity.Mapper); if (mapper == null) return null; return mapper.Text; } public bool DeleteScenario(User user, string scenarioName) { // delete from table & blob store var scenarioDao = dal.ScenarioDao; var blobDao = dal.BlobDao; var entity = scenarioDao.FindByName(scenarioName); if (entity == null) return false; blobDao.DeleteByKey(entity.Mapper); blobDao.DeleteByKey(entity.Scenario); scenarioDao.DeleteByName(scenarioName); return true; } public bool SaveExperiment(User user, Model.Experiment experiment) { return dal.ExperimentDao.Add(user.Username, experiment); } public IEnumerable GetExperiments(User user) { return (from exp in dal.ExperimentDao.GetExperiments(user.Username) select exp.Name); } public bool DeleteExperiment(User user, string experiment) { return dal.ExperimentDao.DeleteByName(user.Username, experiment); } public Model.Task GetTaskData(User u, string jobId, string taskId) { ConfigureHive(u); HiveServiceLocator.Instance.Username = u.Username; HiveServiceLocator.Instance.Password = u.Password; HiveServiceLocator.Instance.EndpointConfigurationName = Configuration.HiveEndpointName; TaskDownloader downloader = new TaskDownloader(new List(){Guid.Parse(taskId)}); downloader.StartAsync(); while (!downloader.IsFinished) { Thread.Sleep(250); if (downloader.IsFaulted) { throw downloader.Exception; } } IDictionary hiveTasks = downloader.Results; var task = hiveTasks[Guid.Parse(taskId)]; if (task == null) return null; return new Model.Task() { State = new Model.TaskState() { DateCreated = task.Task.DateCreated.ToString(), DateFinished = task.Task.DateFinished.ToString(), ExecutionTime = task.Task.ExecutionTime.ToString(), State = task.Task.State.ToString() }, General = new Model.General() { Id = task.Task.Id.ToString(), LastChanged = task.Task.Modified.ToString(), Name = task.ItemTask.Name } }; } public Model.Job GetTasks(User u, string jobId) { var serviceLocator = ConfigureHive(u); var jobTasks = serviceLocator.CallHiveService>(s => s.GetLightweightJobTasks(Guid.Parse(jobId))); var job = new Model.Job(); job.Id = jobId; var tasks = new Dictionary(); // push all elements to dictionary foreach (var task in jobTasks) { // TODO: Crawl children + parent and create hierarchy! var children = serviceLocator.CallHiveService>(s => s.GetLightweightChildTasks(Guid.Parse(jobId), true, true)); foreach (var child in children) { tasks[child.Id] = new Model.Task() { State = new Model.TaskState() { DateCreated = child.DateCreated.ToString(), DateFinished = child.DateFinished.ToString(), ExecutionTime = child.ExecutionTime.ToString(), State = child.State.ToString() }, General = new Model.General() { Id = child.Id.ToString(), LastChanged = child.Modified.ToString(), Name = child.ItemName } }; } } // traverse all tasks again and create tree of tasks foreach (var task in jobTasks) { if (task.ParentTaskId.HasValue) tasks[task.Id].Children.Add(tasks[task.ParentTaskId.Value]); else // its a root task job.Tasks.Add(tasks[task.Id]); } return job; } public Model.Experiment GetExperimentByName(User user, string scenario) { return dal.ExperimentDao.GetExperimentByName(user.Username, scenario); } } }