source: branches/OaaS/HeuristicLab.Services.Optimization.Controller/Azure/DAL.cs @ 9395

Last change on this file since 9395 was 9395, checked in by fschoepp, 9 years ago

#1888:

  • Added visual extensions (dynamic JavaScript) which will be used to render additional result parameters specific to scenarios (e. g. create a graphical representation of a TSP).
  • Added relationship between jobs and experiments (otherwise, it's not possible to get the job's experiment).
  • Updated Admin page to allow removal/addition of visual extensions.
  • Added back-end logic to store/retrieve/delete visual extensions.
  • Added visual extension functionality to the JavaScript views/controllers (job.*.js).
  • Added tsp.js which is a visual extension for the "Genetic Algorithm - TSP" scenario. It adds a graphical representation of the TSP (just like the C# version does) to the results.
File size: 19.2 KB
Line 
1using System;
2using System.Collections.Generic;
3using System.Linq;
4using System.Text;
5using HeuristicLab.Services.Optimization.ControllerService.Interfaces;
6using Microsoft.WindowsAzure;
7using Microsoft.WindowsAzure.StorageClient;
8using Microsoft.WindowsAzure.ServiceRuntime;
9using System.Diagnostics;
10using HeuristicLab.Services.Optimization.ControllerService.Model;
11using HeuristicLab.Services.Optimization.ControllerService.Parsers;
12
13namespace HeuristicLab.Services.Optimization.ControllerService.Azure {
14  public static class AzureConstants {
15    public static readonly string SCENARIO_TABLE = "Scenario";
16    public static readonly string SCENARIO_BLOB_CONTAINER = "scenario";
17    public static readonly string EXPERIMENT_TABLE = "Experiment";
18    public static readonly string EXPERIMENT_BLOB_CONTAINER = "experiment";
19    public static readonly string VISUAL_BLOB_CONTAINER = "visualextensions";
20    public static readonly string JOB_TABLE = "Job";
21    public static readonly string CLOUD_SETTINGS_KEY = "Cloudia.WindowsAzure.Storage";
22
23   
24  }
25
26  public class ScenarioDao : IScenarioDao {   
27    public CloudTableClient TableClient { get; set; }
28
29    public bool Add(ScenarioEntity entity) {
30      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
31      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
32      ScenarioEntity dbEntity = (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
33                               where e.RowKey == entity.RowKey
34                               select e).FirstOrDefault();
35      if (dbEntity != null)
36        return false;
37
38      serviceContext.AddObject(AzureConstants.SCENARIO_TABLE, entity);
39      serviceContext.SaveChanges();
40      return true;
41    }
42
43    public bool DeleteByName(string scenarioName) {
44      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
45      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
46      ScenarioEntity entity = (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
47                               where e.RowKey == scenarioName
48                               select e).FirstOrDefault();
49      if (entity == null)
50        return false;
51
52      serviceContext.DeleteObject(entity);
53      serviceContext.SaveChangesWithRetries();
54      return true;
55    }
56
57    public ScenarioEntity FindByName(string scenarioName) {
58      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
59      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
60      ScenarioEntity entity = (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
61                               where e.RowKey == scenarioName
62                               select e).FirstOrDefault();
63      return entity;
64    }
65
66
67    public IEnumerable<ScenarioEntity> GetAllEntities() {
68      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
69      TableClient.CreateTableIfNotExist(AzureConstants.SCENARIO_TABLE);
70      return (from e in serviceContext.CreateQuery<ScenarioEntity>(AzureConstants.SCENARIO_TABLE)
71             select e).AsEnumerable();
72    }
73  }
74
75  public class BlobDao : IBlobDao {
76    public CloudBlobClient BlobClient { get; set; }
77
78    public bool Add(StringEntry entry) {
79      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER);
80      container.CreateIfNotExist();
81      var blob = container.GetBlobReference(entry.Key);
82      blob.UploadText(entry.Text);
83      return true;
84    }
85
86    public bool DeleteByKey(string entryKey) {
87      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER);
88      container.CreateIfNotExist();
89      var blob = container.GetBlobReference(entryKey);
90      return blob.DeleteIfExists();
91    }
92
93    public StringEntry FindByKey(string entryKey) {
94      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.SCENARIO_BLOB_CONTAINER);
95      container.CreateIfNotExist();
96      var blob = container.GetBlobReference(entryKey);
97      return new StringEntry() { Key = entryKey, Text = blob.DownloadText() };     
98    }
99  }
100
101  internal sealed class ExperimentEntity : TableServiceEntity {
102    public ExperimentEntity() {
103    }
104
105    public ExperimentEntity(string user, string experimentName, string experimentId, string experimentUrl) {
106      PartitionKey = "ScenarioPartition";
107      RowKey = user + "_" + experimentName;     
108      User = user;
109      ExperimentId = experimentId;
110      ExperimentJsonUrl = experimentUrl;
111    }
112
113    public string ExperimentId { get; set; }
114
115    public string User { get; set; }
116
117    public string ExperimentJsonUrl { get; set; }
118
119  }
120
121  public class ExperimentDao : IExperimentDao {
122    public CloudBlobClient BlobClient { get; set; }
123    public CloudTableClient TableClient { get; set; }
124
125    public bool Add(string username, Model.Experiment experiment) {
126      if (FindByName(username, experiment.Name) != null)
127        return false;
128
129      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
130      container.CreateIfNotExist();
131      // For now we store it as JSON element in the blob store
132      var experimentJson = AlgorithmConverter.ConvertExperimentToJson(experiment);
133      Guid experimentJsonGuid = Guid.NewGuid();
134      var experimentJsonId = experiment.Name + "_" + experimentJsonGuid.ToString();
135      experimentJson["nodeId"] = experimentJsonId.ToString();
136      var blob = container.GetBlobReference(experimentJsonId);     
137      blob.UploadText(experimentJson.ToString());
138      experiment.Id = experimentJsonId;
139
140      TableServiceContext serviceContext = TableClient.GetDataServiceContext();     
141      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
142      var entity = new ExperimentEntity(username, experiment.Name, experiment.Id, blob.Uri.ToString());
143      serviceContext.AddObject(AzureConstants.EXPERIMENT_TABLE, entity);
144      serviceContext.SaveChangesWithRetries();
145      return true;
146    }
147
148    public bool Update(string username, Experiment experiment) {
149      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
150      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
151      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
152                    where e.ExperimentId == experiment.Id
153                    select e).FirstOrDefault();
154      if (entity == null) {
155        return false;
156      }
157
158      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
159      container.CreateIfNotExist();
160      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
161      var experimentJson = AlgorithmConverter.ConvertExperimentToJson(experiment).ToString();
162      blob.UploadText(experimentJson);
163      return true;
164    }
165
166    public bool Delete(string username, string experimentId) {
167      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
168      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
169      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
170                    where e.ExperimentId == experimentId
171                    select e).FirstOrDefault();
172
173      if (entity == null)
174        return false;
175
176      if (entity.ExperimentJsonUrl != null) {
177        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
178        container.CreateIfNotExist();
179        var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
180        blob.DeleteIfExists();
181      }
182
183      serviceContext.DeleteObject(entity);
184      serviceContext.SaveChangesWithRetries();
185      return true;
186    }
187
188    public bool DeleteByName(string username, string experiment) {
189      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
190      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
191      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
192                    where e.RowKey == (username + "_" + experiment)
193                    select e).FirstOrDefault();
194     
195      if (entity == null)
196        return false;
197
198      if (entity.ExperimentJsonUrl != null) {
199        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
200        container.CreateIfNotExist();
201        var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
202        blob.DeleteIfExists();
203      }
204
205      serviceContext.DeleteObject(entity);
206      serviceContext.SaveChangesWithRetries();
207      return true;
208    }
209
210    public Model.Experiment FindByName(string username, string experiment) {
211      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
212      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
213      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
214                    where e.RowKey == (username + "_" + experiment)
215                    select e).FirstOrDefault();
216     
217      if (entity == null) {
218        return null;
219      }
220
221      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
222      container.CreateIfNotExist();
223      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
224      return AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText());
225    }
226
227    //private Experiment Convert(ExperimentEntity entity, string entityJson) {
228    //  // TODO: Read the whole experiment, not just the names!
229    //  var exp = new Experiment() { Name = entity.RowKey.Split('_')[1] };
230    //  foreach (var scenarioName in entity.Algorithms.Split(','))
231    //    exp.Algorithm.Add(new Algorithm() { Name = scenarioName });
232    //  return exp;
233    //}
234
235    public IEnumerable<Model.Experiment> GetExperiments(string user, bool namesOnly=false) {
236      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
237      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
238      var entites = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
239                    where e.User == user
240                    select e).ToList();
241      var experiments = new List<Experiment>();
242      if (namesOnly) {
243        return (from e in entites select new Model.Experiment() { Id = e.ExperimentId, Name = e.RowKey.Split('_')[1] });
244      }
245      else {
246        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
247        container.CreateIfNotExist();
248        foreach (var entity in entites) {
249          var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
250          experiments.Add(AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText()));
251        }
252        return experiments;
253      }
254    }
255
256
257    public Experiment GetExperimentByName(string username, string scenario) {
258      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
259      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
260      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
261                      where e.RowKey == username + "_" + scenario
262                      select e).FirstOrDefault();
263      if (entity == null || entity.ExperimentJsonUrl == null) {
264        return null;
265      }
266
267      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
268      container.CreateIfNotExist();
269      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
270      var exp = AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText());
271      return exp;
272    }
273
274
275    public Experiment GetExperimentById(User user, string nodeId) {
276      TableServiceContext serviceContext = TableClient.GetDataServiceContext();
277      TableClient.CreateTableIfNotExist(AzureConstants.EXPERIMENT_TABLE);
278      var entity = (from e in serviceContext.CreateQuery<ExperimentEntity>(AzureConstants.EXPERIMENT_TABLE)
279                    where e.ExperimentId == nodeId
280                    select e).FirstOrDefault();
281      if (entity == null || entity.ExperimentJsonUrl == null) {
282        return null;
283      }
284
285      if (entity.User != user.Username)
286        return null;
287
288      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.EXPERIMENT_BLOB_CONTAINER);
289      container.CreateIfNotExist();
290      var blob = container.GetBlobReference(entity.ExperimentJsonUrl);
291      return AlgorithmConverter.ConvertJsonToExperiment(blob.DownloadText());
292    }
293  }
294
295  public class VisualExtensionDao : IVisualExtensionDao {
296    public CloudBlobClient BlobClient { get; set; }
297
298    public bool Add(string algorithmId, string script) {
299      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
300      container.CreateIfNotExist();
301      var blob = container.GetBlobReference(algorithmId);
302      blob.UploadText(script);
303      return true;
304    }
305
306    public bool DeleteById(string algorithmId) {
307      CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
308      container.CreateIfNotExist();
309      var blob = container.GetBlobReference(algorithmId);
310      return blob.DeleteIfExists();
311    }
312
313    public string FindById(string algorithmId) {
314      try {
315        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
316        container.CreateIfNotExist();
317        var blob = container.GetBlobReference(algorithmId);
318        return blob.DownloadText();
319      }
320      catch (Exception ex) {
321        Trace.TraceError(ex.ToString());
322        return null;
323      }
324    }
325
326
327    public bool Exists(string algorithmId) {
328      try {
329        CloudBlobContainer container = BlobClient.GetContainerReference(AzureConstants.VISUAL_BLOB_CONTAINER);
330        container.CreateIfNotExist();
331        var blob = container.GetBlobReference(algorithmId);
332        blob.FetchAttributes();
333        return true;
334      }
335      catch (StorageClientException ex) {
336        if (ex.ErrorCode == StorageErrorCode.ResourceNotFound) {
337          return false;
338        }
339        Trace.TraceError(ex.ToString());
340        return false;
341      }
342    }
343  }
344
345  internal sealed class JobEntity : TableServiceEntity {
346    public JobEntity() {
347    }
348
349    public JobEntity(string user, string experimentName, string experimentId, string jobId) {
350      PartitionKey = "JobPartition";
351      RowKey = user + "_" + jobId;
352      User = user;
353      ExperimentId = experimentId;
354      JobId = jobId;
355    }
356
357    public string ExperimentId { get; set; }
358
359    public string User { get; set; }
360
361    public string JobId { get; set; }
362  }
363
364  public class JobDao : IJobDao {
365    public IExperimentDao ExperimentDao { get; set; }
366    public CloudTableClient TableClient { get; set; }
367
368    public bool Add(string username, Experiment experiment, string jobId) {
369      try {
370        TableServiceContext serviceContext = TableClient.GetDataServiceContext();
371        TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE);
372        serviceContext.AddObject(AzureConstants.JOB_TABLE,
373          new JobEntity(username, experiment.Name, experiment.Id, jobId)
374        );
375        serviceContext.SaveChangesWithRetries();
376        return true;
377      }
378      catch (Exception ex) {
379        Trace.TraceError(ex.ToString());
380        return false;
381      }
382    }
383
384    public bool Delete(string username, string jobId) {
385      try {
386        TableServiceContext serviceContext = TableClient.GetDataServiceContext();
387        TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE);
388        var entity = (from e in serviceContext.CreateQuery<JobEntity>(AzureConstants.JOB_TABLE)
389                      where e.JobId == jobId && e.User == username
390                      select e).FirstOrDefault();
391        serviceContext.DeleteObject(entity);       
392        return true;
393      }
394      catch (Exception ex) {
395        Trace.TraceError(ex.ToString());
396        return false;
397      }
398    }
399
400    public Experiment FindByJobId(string username, string jobId) {
401      try {
402        TableServiceContext serviceContext = TableClient.GetDataServiceContext();
403        TableClient.CreateTableIfNotExist(AzureConstants.JOB_TABLE);
404        var entity = (from e in serviceContext.CreateQuery<JobEntity>(AzureConstants.JOB_TABLE)
405                      where e.JobId == jobId && e.User == username
406                      select e).FirstOrDefault();
407        return ExperimentDao.GetExperimentById(new User() { Username = username }, entity.ExperimentId);
408      }
409      catch (Exception ex) {
410        Trace.TraceError(ex.ToString());
411        return null;
412      }
413    }
414  }
415
416  public class AzureDataAccessLayer : IDataAccessLayer {
417    private IScenarioDao scenarioDao;
418    private IBlobDao blobDao;
419    private IExperimentDao expDao;
420    private IVisualExtensionDao visualDao;
421    private IJobDao jobDao;
422
423    private CloudStorageAccount storageAccount;
424
425    private CloudStorageAccount StorageAccount {
426      get {
427        if (storageAccount == null) {
428          try {
429            storageAccount = CloudStorageAccount.Parse(CloudConfigurationManager.GetSetting(AzureConstants.CLOUD_SETTINGS_KEY));
430          }
431          catch (Exception ex) {
432            Trace.WriteLine(ex.Message);
433            storageAccount = CloudStorageAccount.Parse("DefaultEndpointsProtocol=https;AccountName=optimizationstorage1;AccountKey=n7Leom8ZFWkof/VQ2a4aRSvwOlX+Gwr3uojQF9CFJw1osmGCV0WwaNC8s7nkZ+qteLduAgW2l75WFpbXrkvG4Q==");
434          }
435        }
436        return storageAccount;
437      }     
438    }
439
440    public IScenarioDao ScenarioDao {
441      get {
442        if (scenarioDao == null) {
443          scenarioDao = new ScenarioDao() { TableClient = StorageAccount.CreateCloudTableClient() };
444        }
445        return scenarioDao;
446      }
447    }
448
449    public IBlobDao BlobDao {
450      get {
451        if (blobDao == null) {
452          blobDao = new BlobDao() { BlobClient = StorageAccount.CreateCloudBlobClient() };
453        }
454        return blobDao;
455      }
456    }
457
458    public IExperimentDao ExperimentDao {
459      get {
460        if (expDao == null) {
461          expDao = new ExperimentDao() { TableClient = StorageAccount.CreateCloudTableClient(), BlobClient = StorageAccount.CreateCloudBlobClient() };
462        }
463        return expDao;
464      }
465    }
466
467
468    public IVisualExtensionDao VisualExtensionDao {
469      get {
470        if (visualDao == null) {
471          visualDao = new VisualExtensionDao() { BlobClient = StorageAccount.CreateCloudBlobClient() };
472        }
473        return visualDao;
474      }
475    }
476
477
478    public IJobDao JobDao {
479      get {
480        if (jobDao == null) {
481          jobDao = new JobDao() { ExperimentDao = ExperimentDao, TableClient = StorageAccount.CreateCloudTableClient() };
482        }
483        return jobDao;
484      }
485    }
486  }
487}
Note: See TracBrowser for help on using the repository browser.