source: branches/OaaS/HeuristicLab.Services.Optimization.Controller/Azure/DAL.cs @ 13749

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using HeuristicLab.Services.Optimization.ControllerService.Interfaces;
using Microsoft.WindowsAzure;
using Microsoft.WindowsAzure.StorageClient;
using Microsoft.WindowsAzure.ServiceRuntime;
using System.Diagnostics;
using HeuristicLab.Services.Optimization.ControllerService.Model;
using HeuristicLab.Services.Optimization.ControllerService.Parsers;

namespace HeuristicLab.Services.Optimization.ControllerService.Azure {
  public static class AzureConstants {
    public static readonly string SCENARIO_TABLE = "Scenario";
    public static readonly string SCENARIO_BLOB_CONTAINER = "scenario";
    public static readonly string EXPERIMENT_TABLE = "Experiment";
    public static readonly string EXPERIMENT_BLOB_CONTAINER = "experiment";
    public static readonly string VISUAL_BLOB_CONTAINER = "visualextensions";
    public static readonly string JOB_TABLE = "Job";
    public static readonly string CLOUD_SETTINGS_KEY = "Cloudia.WindowsAzure.Storage";

    
  }

  public class ScenarioDao : IScenarioDao {    
    public CloudTableClient TableClient { get; set; }

    public bool Add(ScenarioEntity entity) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
      ScenarioEntity dbEntity = (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
                               where e.RowKey == entity.RowKey
                               select e).FirstOrDefault();
      if (dbEntity != null)
        return false;

      serviceContext.AddObject(AzureConstants.SCENARIO_TABLE, entity);
      serviceContext.SaveChanges();
      return true;
    }

    public bool DeleteByName(string scenarioName) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
      ScenarioEntity entity = (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
                               where e.RowKey == scenarioName
                               select e).FirstOrDefault();
      if (entity == null)
        return false;

      serviceContext.DeleteObject(entity);
      serviceContext.SaveChangesWithRetries();
      return true;
    }

    public ScenarioEntity FindByName(string scenarioName) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
      ScenarioEntity entity = (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
                               where e.RowKey == scenarioName
                               select e).FirstOrDefault();
      return entity;
    }


    public IEnumerable<ScenarioEntity> GetAllEntities() {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
      return (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
             select e).AsEnumerable();
    }
  }

  public class BlobDao : IBlobDao {
    public CloudBlobClient BlobClient { get; set; }

    public bool Add(StringEntry entry) {
      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(entry.Key);
      blob.UploadText(entry.Text);
      return true;
    }

    public bool DeleteByKey(string entryKey) {
      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(entryKey);
      return blob.DeleteIfExists();
    }

    public StringEntry FindByKey(string entryKey) {
      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(entryKey);
      return new StringEntry() { Key = entryKey, Text = blob.DownloadText() };      
    }
  }

  internal sealed class ExperimentEntity : TableServiceEntity {
    public ExperimentEntity() {
    }

    public ExperimentEntity(string user, string experimentName, string experimentId, string experimentUrl) {
      PartitionKey = "ScenarioPartition";
      RowKey = user + "_" + experimentName;      
      User = user;
      ExperimentId = experimentId;
      ExperimentJsonUrl = experimentUrl;
    }

    public string ExperimentId { get; set; }

    public string User { get; set; }

    public string ExperimentJsonUrl { get; set; }

  }

  public class ExperimentDao : IExperimentDao {
    public CloudBlobClient BlobClient { get; set; }
    public CloudTableClient TableClient { get; set; }

    public bool Add(string username, Model.Experiment experiment) {
      if (FindByName(username, experiment.Name) != null)
        return false;

      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
      container.CreateIfNotExist();
      // For now we store it as JSON element in the blob store
      var experimentJson = AlgorithmConverter.ConvertExperimentToJson(experiment);
      Guid experimentJsonGuid = Guid.NewGuid();
      var experimentJsonId = experiment.Name + "_" + experimentJsonGuid.ToString();
      experimentJson["nodeId"] = experimentJsonId.ToString();
      var blob = container.GetBlobReference(experimentJsonId);      
      blob.UploadText(experimentJson.ToString());
      experiment.Id = experimentJsonId;

      TableServiceContext serviceContext = TableClient.GetDataServiceContext();      
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entity = new ExperimentEntity(username, experiment.Name, experiment.Id, blob.Uri.ToString());
      serviceContext.AddObject(AzureConstants.EXPERIMENT_TABLE, entity);
      serviceContext.SaveChangesWithRetries();
      return true;
    }

    public bool Update(string username, Experiment experiment) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
                    where e.ExperimentId == experiment.Id
                    select e).FirstOrDefault();
      if (entity == null) {
        return false;
      }

      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
      var experimentJson = AlgorithmConverter.ConvertExperimentToJson(experiment).ToString();
      blob.UploadText(experimentJson);
      return true;
    }

    public bool Delete(string username, string experimentId) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
                    where e.ExperimentId == experimentId
                    select e).FirstOrDefault();

      if (entity == null)
        return false;

      if (entity.ExperimentJsonUrl != null) {
        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
        container.CreateIfNotExist();
        var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
        blob.DeleteIfExists();
      }

      serviceContext.DeleteObject(entity);
      serviceContext.SaveChangesWithRetries();
      return true;
    }

    public bool DeleteByName(string username, string experiment) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
                    where e.RowKey == (username + "_" + experiment)
                    select e).FirstOrDefault();
      
      if (entity == null)
        return false;

      if (entity.ExperimentJsonUrl != null) {
        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
        container.CreateIfNotExist();
        var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
        blob.DeleteIfExists();
      }

      serviceContext.DeleteObject(entity);
      serviceContext.SaveChangesWithRetries();
      return true;
    }

    public Model.Experiment FindByName(string username, string experiment) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
                    where e.RowKey == (username + "_" + experiment)
                    select e).FirstOrDefault();
      
      if (entity == null) {
        return null;
      }

      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
      return AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText());
    }

    public IEnumerable<Model.Experiment> GetExperiments(string user, bool namesOnly=false) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entites = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
                    where e.User == user
                    select e).ToList();
      var experiments = new List<Experiment>();
      if (namesOnly) {
        return (from e in entites select new Model.Experiment() { Id = e.ExperimentId, Name = e.RowKey.Split('_')[1] });
      }
      else {
        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
        container.CreateIfNotExist();
        foreach (var entity in entites) {
          var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
          experiments.Add(AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText()));
        }
        return experiments;
      }
    }


    public Experiment GetExperimentByName(string username, string scenario) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
                      where e.RowKey == username + "_" + scenario
                      select e).FirstOrDefault();
      if (entity == null || entity.ExperimentJsonUrl == null) {
        return null;
      }

      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
      var exp = AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText());
      return exp;
    }


    public Experiment GetExperimentById(User user, string nodeId) {
      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
                    where e.ExperimentId == nodeId
                    select e).FirstOrDefault();
      if (entity == null || entity.ExperimentJsonUrl == null) {
        return null;
      }

      if (entity.User != user.Username)
        return null;

      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
      return AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText());
    }
  }

  public class VisualExtensionDao : IVisualExtensionDao {
    public CloudBlobClient BlobClient { get; set; }

    public bool Add(string algorithmId, string script) {
      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(algorithmId);
      blob.UploadText(script);
      return true;
    }

    public bool DeleteById(string algorithmId) {
      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
      container.CreateIfNotExist();
      var blob = container.GetBlobReference(algorithmId);
      return blob.DeleteIfExists();
    }

    public string FindById(string algorithmId) {
      try {
        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
        container.CreateIfNotExist();
        var blob = container.GetBlobReference(algorithmId);
        return blob.DownloadText();
      }
      catch (Exception ex) {
        Trace.TraceError(ex.ToString());
        return null;
      }
    }


    public bool Exists(string algorithmId) {
      try {
        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
        container.CreateIfNotExist();
        var blob = container.GetBlobReference(algorithmId);
        blob.FetchAttributes();
        return true;
      }
      catch (StorageClientException ex) {
        if (ex.ErrorCode == StorageErrorCode.ResourceNotFound) {
          return false;
        }
        Trace.TraceError(ex.ToString());
        return false;
      }
    }
  }

  internal sealed class JobEntity : TableServiceEntity {
    public JobEntity() {
    }

    public JobEntity(string user, string experimentName, string experimentId, string jobId) {
      PartitionKey = "JobPartition";
      RowKey = user + "_" + jobId;
      User = user;
      ExperimentId = experimentId;
      JobId = jobId;
    }

    public string ExperimentId { get; set; }

    public string User { get; set; }

    public string JobId { get; set; }
  }

  public class JobDao : IJobDao {
    public IExperimentDao ExperimentDao { get; set; }
    public CloudTableClient TableClient { get; set; }

    public bool Add(string username, Experiment experiment, string jobId) {
      try {
        TableServiceContext serviceContext = TableClient.GetDataServiceContext();
        TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE);
        serviceContext.AddObject(AzureConstants.JOB_TABLE,
          new JobEntity(username, experiment.Name, experiment.Id, jobId)
        );
        serviceContext.SaveChangesWithRetries();
        return true;
      }
      catch (Exception ex) {
        Trace.TraceError(ex.ToString());
        return false;
      }
    }

    public bool Delete(string username, string jobId) {
      try {
        TableServiceContext serviceContext = TableClient.GetDataServiceContext();
        TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE);
        var entity = (from e in serviceContext.CreateQuery<JobEntity>(AzureConstants.JOB_TABLE)
                      where e.JobId == jobId && e.User == username
                      select e).FirstOrDefault();
        serviceContext.DeleteObject(entity);        
        return true;
      }
      catch (Exception ex) {
        Trace.TraceError(ex.ToString());
        return false;
      }
    }

    public Experiment FindByJobId(string username, string jobId) {
      try {
        TableServiceContext serviceContext = TableClient.GetDataServiceContext();
        TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE);
        var entity = (from e in serviceContext.CreateQuery<JobEntity>(AzureConstants.JOB_TABLE)
                      where e.JobId == jobId && e.User == username
                      select e).FirstOrDefault();
        return ExperimentDao.GetExperimentById(new User() { Username = username }, entity.ExperimentId);
      }
      catch (Exception ex) {
        Trace.TraceError(ex.ToString());
        return null;
      }
    }
  }

  public class AzureDataAccessLayer : IDataAccessLayer {
    private IScenarioDao scenarioDao;
    private IBlobDao blobDao;
    private IExperimentDao expDao;
    private IVisualExtensionDao visualDao;
    private IJobDao jobDao;

    private CloudStorageAccount storageAccount;

    private CloudStorageAccount StorageAccount {
      get {
        if (storageAccount == null) {
          try {
            storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(AzureConstants.CLOUD_SETTINGS_KEY));
          }
          catch (Exception ex) {
            Trace.WriteLine(ex.Message);
            storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=optimizationstorage1;AccountKey=n7Leom8ZFWkof/VQ2a4aRSvwOlX+Gwr3uojQF9CFJw1osmGCV0WwaNC8s7nkZ+qteLduAgW2l75WFpbXrkvG4Q==");
          }
        }
        return storageAccount; 
      }      
    }

    public IScenarioDao ScenarioDao {
      get {
        if (scenarioDao == null) {
          scenarioDao = new ScenarioDao() { TableClient = StorageAccount.CreateCloudTableClient() };
        }
        return scenarioDao;
      }
    }

    public IBlobDao BlobDao {
      get {
        if (blobDao == null) {
          blobDao = new BlobDao() { BlobClient = StorageAccount.CreateCloudBlobClient() };
        }
        return blobDao;
      }
    }

    public IExperimentDao ExperimentDao {
      get {
        if (expDao == null) {
          expDao = new ExperimentDao() { TableClient = StorageAccount.CreateCloudTableClient(), BlobClient = StorageAccount.CreateCloudBlobClient() };
        }
        return expDao;
      }
    }


    public IVisualExtensionDao VisualExtensionDao {
      get {
        if (visualDao == null) {
          visualDao = new VisualExtensionDao() { BlobClient = StorageAccount.CreateCloudBlobClient() };
        }
        return visualDao;
      }
    }


    public IJobDao JobDao {
      get {
        if (jobDao == null) {
          jobDao = new JobDao() { ExperimentDao = ExperimentDao, TableClient = StorageAccount.CreateCloudTableClient() };
        }
        return jobDao;
      }
    }
  }
}