Free cookie consent management tool by TermsFeed Policy Generator

source: branches/HeuristicLab.Hive.Azure/HeuristicLab.Clients.Hive.CloudManager/3.3/Azure/CloudBlobExtension.cs @ 8899

Last change on this file since 8899 was 7563, checked in by spimming, 13 years ago

#1680:

  • Create deployment with local configuration file
  • New extension method to parallel upload a file to blob storage
  • Constants for deployment packages and configuration added
File size: 3.1 KB
Line 
1
2using System;
3using System.Collections.Generic;
4using System.IO;
5using System.Text;
6using System.Threading.Tasks;
7using Microsoft.WindowsAzure.StorageClient;
8namespace HeuristicLab.Clients.Hive.CloudManager.Azure {
9  public static class CloudBlobExtension {
10    private class BlockTransferDetail {
11      public int StartPosition { get; set; }
12      public int BytesToRead { get; set; }
13      public string BlockId { get; set; }
14    }
15
16    //check if a blob exists
17    // see: http://stackoverflow.com/questions/2642919/checking-if-a-blob-exists-in-azure-storage
18    public static bool Exists(this CloudBlob blob) {
19      try {
20        blob.FetchAttributes();
21        return true;
22      }
23      catch (StorageClientException ex) {
24        if (ex.ErrorCode == StorageErrorCode.ResourceNotFound) {
25          return false;
26        } else {
27          throw;
28        }
29      }
30    }
31
32    public static void UploadParallel(this CloudBlockBlob blob, string filePath) {
33      FileInfo file = new FileInfo(filePath);
34      //CloudBlockBlob blob = blobContainer.GetBlockBlobReference(file.Name);
35      blob.Properties.ContentType = "application/octet-stream";
36
37      // init
38      int blockSize = 262144;
39      long fileSize = file.Length;
40      long leftToRead = fileSize;
41      int startPosition = 0;
42
43      int blockCount = ((int)Math.Floor((double)(fileSize / blockSize))) + 1;
44      BlockTransferDetail[] transferDetails = new BlockTransferDetail[blockCount];
45      List<string> blockIdList = new List<string>();
46
47      // setup control array
48      for (int j = 0; j < transferDetails.Length; j++) {
49        int toRead = (int)(leftToRead > blockSize ? blockSize : leftToRead);
50        string blockId = Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(string.Format("BlockId{0}", j.ToString("0000000"))));
51        transferDetails[j] = new BlockTransferDetail() { StartPosition = startPosition, BytesToRead = toRead, BlockId = blockId };
52        if (toRead > 0) {
53          blockIdList.Add(blockId);
54        }
55        startPosition += toRead;
56        leftToRead -= toRead;
57      }
58
59      // perform parallel upload
60      var result = Parallel.For(0, transferDetails.Length, j => {
61        using (FileStream fs = new FileStream(file.FullName, FileMode.Open, FileAccess.Read)) {
62          byte[] buffer = new byte[transferDetails[j].BytesToRead];
63          BinaryReader br = new BinaryReader(fs);
64          fs.Seek(transferDetails[j].StartPosition, SeekOrigin.Begin);
65          br.Read(buffer, 0, transferDetails[j].BytesToRead);
66          if (buffer.Length > 0) {
67            using (MemoryStream ms = new MemoryStream(buffer)) {
68              blob.PutBlock(transferDetails[j].BlockId, ms, null);
69            }
70          }
71        }
72      });
73      blob.PutBlockList(blockIdList);
74    }
75
76    public static bool Exists(this CloudBlobContainer blob) {
77      try {
78        blob.FetchAttributes();
79        return true;
80      }
81      catch (StorageClientException ex) {
82        if (ex.ErrorCode == StorageErrorCode.ResourceNotFound) {
83          return false;
84        } else {
85          throw;
86        }
87      }
88    }
89  }
90}
Note: See TracBrowser for help on using the repository browser.