using System; using System.Collections.Generic; using System.Linq; using System.Text; using HeuristicLab.Services.Optimization.ControllerService.Interfaces; using Microsoft.WindowsAzure; using Microsoft.WindowsAzure.StorageClient; using Microsoft.WindowsAzure.ServiceRuntime; using System.Diagnostics; using HeuristicLab.Services.Optimization.ControllerService.Model; using HeuristicLab.Services.Optimization.ControllerService.Parsers; namespace HeuristicLab.Services.Optimization.ControllerService.Azure { public static class AzureConstants { public static readonly string SCENARIO_TABLE = "Scenario"; public static readonly string SCENARIO_BLOB_CONTAINER = "scenario"; public static readonly string EXPERIMENT_TABLE = "Experiment"; public static readonly string EXPERIMENT_BLOB_CONTAINER = "experiment"; public static readonly string VISUAL_BLOB_CONTAINER = "visualextensions"; public static readonly string JOB_TABLE = "Job"; public static readonly string CLOUD_SETTINGS_KEY = "Cloudia.WindowsAzure.Storage"; } public class ScenarioDao : IScenarioDao { public CloudTableClient TableClient { get; set; } public bool Add(ScenarioEntity entity) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE); ScenarioEntity dbEntity = (from e in serviceContext.CreateQuery(AzureConstants.SCENARIO_TABLE) where e.RowKey == entity.RowKey select e).FirstOrDefault(); if (dbEntity != null) return false; serviceContext.AddObject(AzureConstants.SCENARIO_TABLE, entity); serviceContext.SaveChanges(); return true; } public bool DeleteByName(string scenarioName) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE); ScenarioEntity entity = (from e in serviceContext.CreateQuery(AzureConstants.SCENARIO_TABLE) where e.RowKey == scenarioName select e).FirstOrDefault(); if (entity == null) return false; serviceContext.DeleteObject(entity); serviceContext.SaveChangesWithRetries(); return true; } public ScenarioEntity FindByName(string scenarioName) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE); ScenarioEntity entity = (from e in serviceContext.CreateQuery(AzureConstants.SCENARIO_TABLE) where e.RowKey == scenarioName select e).FirstOrDefault(); return entity; } public IEnumerable GetAllEntities() { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE); return (from e in serviceContext.CreateQuery(AzureConstants.SCENARIO_TABLE) select e).AsEnumerable(); } } public class BlobDao : IBlobDao { public CloudBlobClient BlobClient { get; set; } public bool Add(StringEntry entry) { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entry.Key); blob.UploadText(entry.Text); return true; } public bool DeleteByKey(string entryKey) { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entryKey); return blob.DeleteIfExists(); } public StringEntry FindByKey(string entryKey) { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entryKey); return new StringEntry() { Key = entryKey, Text = blob.DownloadText() }; } } internal sealed class ExperimentEntity : TableServiceEntity { public ExperimentEntity() { } public ExperimentEntity(string user, string experimentName, string experimentId, string experimentUrl) { PartitionKey = "ScenarioPartition"; RowKey = user + "_" + experimentName; User = user; ExperimentId = experimentId; ExperimentJsonUrl = experimentUrl; } public string ExperimentId { get; set; } public string User { get; set; } public string ExperimentJsonUrl { get; set; } } public class ExperimentDao : IExperimentDao { public CloudBlobClient BlobClient { get; set; } public CloudTableClient TableClient { get; set; } public bool Add(string username, Model.Experiment experiment) { if (FindByName(username, experiment.Name) != null) return false; CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); // For now we store it as JSON element in the blob store var experimentJson = AlgorithmConverter.ConvertExperimentToJson(experiment); Guid experimentJsonGuid = Guid.NewGuid(); var experimentJsonId = experiment.Name + "_" + experimentJsonGuid.ToString(); experimentJson["nodeId"] = experimentJsonId.ToString(); var blob = container.GetBlobReference(experimentJsonId); blob.UploadText(experimentJson.ToString()); experiment.Id = experimentJsonId; TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entity = new ExperimentEntity(username, experiment.Name, experiment.Id, blob.Uri.ToString()); serviceContext.AddObject(AzureConstants.EXPERIMENT_TABLE, entity); serviceContext.SaveChangesWithRetries(); return true; } public bool Update(string username, Experiment experiment) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.EXPERIMENT_TABLE) where e.ExperimentId == experiment.Id select e).FirstOrDefault(); if (entity == null) { return false; } CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entity.ExperimentJsonUrl); var experimentJson = AlgorithmConverter.ConvertExperimentToJson(experiment).ToString(); blob.UploadText(experimentJson); return true; } public bool Delete(string username, string experimentId) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.EXPERIMENT_TABLE) where e.ExperimentId == experimentId select e).FirstOrDefault(); if (entity == null) return false; if (entity.ExperimentJsonUrl != null) { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entity.ExperimentJsonUrl); blob.DeleteIfExists(); } serviceContext.DeleteObject(entity); serviceContext.SaveChangesWithRetries(); return true; } public bool DeleteByName(string username, string experiment) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.EXPERIMENT_TABLE) where e.RowKey == (username + "_" + experiment) select e).FirstOrDefault(); if (entity == null) return false; if (entity.ExperimentJsonUrl != null) { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entity.ExperimentJsonUrl); blob.DeleteIfExists(); } serviceContext.DeleteObject(entity); serviceContext.SaveChangesWithRetries(); return true; } public Model.Experiment FindByName(string username, string experiment) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.EXPERIMENT_TABLE) where e.RowKey == (username + "_" + experiment) select e).FirstOrDefault(); if (entity == null) { return null; } CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entity.ExperimentJsonUrl); return AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText()); } public IEnumerable GetExperiments(string user, bool namesOnly=false) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entites = (from e in serviceContext.CreateQuery(AzureConstants.EXPERIMENT_TABLE) where e.User == user select e).ToList(); var experiments = new List(); if (namesOnly) { return (from e in entites select new Model.Experiment() { Id = e.ExperimentId, Name = e.RowKey.Split('_')[1] }); } else { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); foreach (var entity in entites) { var blob = container.GetBlobReference(entity.ExperimentJsonUrl); experiments.Add(AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText())); } return experiments; } } public Experiment GetExperimentByName(string username, string scenario) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.EXPERIMENT_TABLE) where e.RowKey == username + "_" + scenario select e).FirstOrDefault(); if (entity == null || entity.ExperimentJsonUrl == null) { return null; } CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entity.ExperimentJsonUrl); var exp = AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText()); return exp; } public Experiment GetExperimentById(User user, string nodeId) { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.EXPERIMENT_TABLE) where e.ExperimentId == nodeId select e).FirstOrDefault(); if (entity == null || entity.ExperimentJsonUrl == null) { return null; } if (entity.User != user.Username) return null; CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(entity.ExperimentJsonUrl); return AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText()); } } public class VisualExtensionDao : IVisualExtensionDao { public CloudBlobClient BlobClient { get; set; } public bool Add(string algorithmId, string script) { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(algorithmId); blob.UploadText(script); return true; } public bool DeleteById(string algorithmId) { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(algorithmId); return blob.DeleteIfExists(); } public string FindById(string algorithmId) { try { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(algorithmId); return blob.DownloadText(); } catch (Exception ex) { Trace.TraceError(ex.ToString()); return null; } } public bool Exists(string algorithmId) { try { CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER); container.CreateIfNotExist(); var blob = container.GetBlobReference(algorithmId); blob.FetchAttributes(); return true; } catch (StorageClientException ex) { if (ex.ErrorCode == StorageErrorCode.ResourceNotFound) { return false; } Trace.TraceError(ex.ToString()); return false; } } } internal sealed class JobEntity : TableServiceEntity { public JobEntity() { } public JobEntity(string user, string experimentName, string experimentId, string jobId) { PartitionKey = "JobPartition"; RowKey = user + "_" + jobId; User = user; ExperimentId = experimentId; JobId = jobId; } public string ExperimentId { get; set; } public string User { get; set; } public string JobId { get; set; } } public class JobDao : IJobDao { public IExperimentDao ExperimentDao { get; set; } public CloudTableClient TableClient { get; set; } public bool Add(string username, Experiment experiment, string jobId) { try { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE); serviceContext.AddObject(AzureConstants.JOB_TABLE, new JobEntity(username, experiment.Name, experiment.Id, jobId) ); serviceContext.SaveChangesWithRetries(); return true; } catch (Exception ex) { Trace.TraceError(ex.ToString()); return false; } } public bool Delete(string username, string jobId) { try { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.JOB_TABLE) where e.JobId == jobId && e.User == username select e).FirstOrDefault(); serviceContext.DeleteObject(entity); return true; } catch (Exception ex) { Trace.TraceError(ex.ToString()); return false; } } public Experiment FindByJobId(string username, string jobId) { try { TableServiceContext serviceContext = TableClient.GetDataServiceContext(); TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE); var entity = (from e in serviceContext.CreateQuery(AzureConstants.JOB_TABLE) where e.JobId == jobId && e.User == username select e).FirstOrDefault(); return ExperimentDao.GetExperimentById(new User() { Username = username }, entity.ExperimentId); } catch (Exception ex) { Trace.TraceError(ex.ToString()); return null; } } } public class AzureDataAccessLayer : IDataAccessLayer { private IScenarioDao scenarioDao; private IBlobDao blobDao; private IExperimentDao expDao; private IVisualExtensionDao visualDao; private IJobDao jobDao; private CloudStorageAccount storageAccount; private CloudStorageAccount StorageAccount { get { if (storageAccount == null) { try { storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(AzureConstants.CLOUD_SETTINGS_KEY)); } catch (Exception ex) { Trace.WriteLine(ex.Message); storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=optimizationstorage1;AccountKey=n7Leom8ZFWkof/VQ2a4aRSvwOlX+Gwr3uojQF9CFJw1osmGCV0WwaNC8s7nkZ+qteLduAgW2l75WFpbXrkvG4Q=="); } } return storageAccount; } } public IScenarioDao ScenarioDao { get { if (scenarioDao == null) { scenarioDao = new ScenarioDao() { TableClient = StorageAccount.CreateCloudTableClient() }; } return scenarioDao; } } public IBlobDao BlobDao { get { if (blobDao == null) { blobDao = new BlobDao() { BlobClient = StorageAccount.CreateCloudBlobClient() }; } return blobDao; } } public IExperimentDao ExperimentDao { get { if (expDao == null) { expDao = new ExperimentDao() { TableClient = StorageAccount.CreateCloudTableClient(), BlobClient = StorageAccount.CreateCloudBlobClient() }; } return expDao; } } public IVisualExtensionDao VisualExtensionDao { get { if (visualDao == null) { visualDao = new VisualExtensionDao() { BlobClient = StorageAccount.CreateCloudBlobClient() }; } return visualDao; } } public IJobDao JobDao { get { if (jobDao == null) { jobDao = new JobDao() { ExperimentDao = ExperimentDao, TableClient = StorageAccount.CreateCloudTableClient() }; } return jobDao; } } } }