String getHash(String remoteFolderName, String filePath) { String fileName = new File(filePath).getName() CloudBlockBlob blob = container.getBlockBlobReference(remoteFolderName+"/"+filePath) if(!blob) return "" blob.downloadAttributes() byte[] hash = Base64.decode(blob.getProperties().getContentMD5()) BigInteger bigInt = new BigInteger(1, hash) return bigInt.toString(16).padLeft(32, '0') }
public Map<String, Long> getHashMap(long id) throws IOException { String haName = EncyptUtils.encHashArchiveName(id, Main.chunkStoreEncryptionEnabled); try { CloudBlockBlob kblob = container.getBlockBlobReference("keys/" + haName); kblob.downloadAttributes(null, null, opContext); String[] ks = this.getStrings(kblob); HashMap<String, Long> m = new HashMap<String, Long>(ks.length); for (String k : ks) { String[] kv = k.split(":"); m.put(kv[0], Long.parseLong(kv[1])); } return m; } catch (Exception e) { throw new IOException(e); } }
@Override public HashMap<String, String> getUserMetaData(String object) throws IOException { try { CloudBlockBlob blob = container.getBlockBlobReference(object); blob.downloadAttributes(); return blob.getMetadata(); } catch (Exception e) { SDFSLogger.getLog().info("unable to download attribute", e); throw new IOException(e); } }
@Override public void updateBucketInfo(Map<String, String> md) throws IOException { try { String lbi = "bucketinfo/" + EncyptUtils.encHashArchiveName(Main.DSEID, Main.chunkStoreEncryptionEnabled); CloudBlockBlob blob = container.getBlockBlobReference(lbi); blob.downloadAttributes(); blob.setMetadata((HashMap<String, String>) md); blob.uploadText(Long.toString(System.currentTimeMillis())); blob.uploadMetadata(null, null, opContext); } catch (Exception e) { throw new IOException(e); } }
private int getClaimedObjects(CloudBlockBlob blob, long id) throws IOException { try { String[] hs = this.getStrings(blob); HashMap<String, String> md = blob.getMetadata(); if (!md.containsKey("encrypt")) { blob.downloadAttributes(); md = blob.getMetadata(); } int claims = 0; for (String ha : hs) { byte[] b = BaseEncoding.base64().decode(ha.split(":")[0]); if (HCServiceProxy.getHashesMap().mightContainKey(b, id)) claims++; } return claims; } catch (Exception e) { throw new IOException(e); } }
private String[] getStrings(CloudBlockBlob blob) throws StorageException, IOException { HashMap<String, String> md = blob.getMetadata(); byte[] nm = new byte[(int) blob.getProperties().getLength()]; blob.downloadToByteArray(nm, 0, null, null, opContext); if (!md.containsKey("encrypt")) { blob.downloadAttributes(); md = blob.getMetadata(); } boolean encrypt = Boolean.parseBoolean(md.get("encrypt")); if (encrypt) { nm = EncryptUtils.decryptCBC(nm); } boolean compress = Boolean.parseBoolean(md.get("lz4Compress")); if (compress) { int size = Integer.parseInt(md.get("size")); nm = CompressionUtils.decompressLz4(nm, size); } String st = new String(nm); return st.split(","); }
+ EncyptUtils.encHashArchiveName(Main.DSEID, Main.chunkStoreEncryptionEnabled); CloudBlockBlob blob = container.getBlockBlobReference(lbi); blob.downloadAttributes(); HashMap<String, String> md = blob.getMetadata(); Long.parseLong(md.get("currentlength"));
@Override public Map<String, String> getBucketInfo() { try { String lbi = "bucketinfo/" + EncyptUtils.encHashArchiveName(Main.DSEID, Main.chunkStoreEncryptionEnabled); CloudBlockBlob blob = container.getBlockBlobReference(lbi); blob.downloadAttributes(); HashMap<String, String> md = blob.getMetadata(); return md; } catch (Exception e) { SDFSLogger.getLog().warn("unable to update metadata for bucket" + Main.DSEID, e); return null; } }
@Override protected Map<String, String> doGetMeta(String fullPath) { try { CloudBlockBlob blob = blobContainer.getBlockBlobReference(fullPath); blob.downloadAttributes(); Map<String, String> meta = C.newMap(blob.getMetadata()); meta.put(ISObject.ATTR_CONTENT_TYPE, blob.getProperties().getContentType()); return meta; } catch (Exception e) { throw E.unexpected(e); } }
@Override public void checkoutObject(long id, int claims) throws IOException { try { CloudBlockBlob cblob = container.getBlockBlobReference(this.getClaimName(id)); if (cblob.exists()) return; else { String haName = EncyptUtils.encHashArchiveName(id, Main.chunkStoreEncryptionEnabled); CloudBlockBlob kblob = container.getBlockBlobReference("keys/" + haName); kblob.downloadAttributes(); HashMap<String, String> metaData = kblob.getMetadata(); cblob.setMetadata(metaData); cblob.uploadText(Long.toString(System.currentTimeMillis())); } } catch (Exception e) { throw new IOException(e); } }
public static void main(String[] args) { String storageConnectionString = "DefaultEndpointsProtocol=http;" + "AccountName=" + args[0] + ";" + "AccountKey=" + args[1]; try { // Retrieve storage account from connection-string. CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); // Create the blob client. CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); // Get a reference to a container. // The container name must be lower case CloudBlobContainer container = blobClient.getContainerReference(args[2]); CloudBlockBlob kblob = container.getBlockBlobReference("blocks/MTEwMjE4ODc4MjM0MzY3NzM="); kblob.downloadAttributes(); System.out.println(kblob.getProperties().getStandardBlobTier()); System.out.println(kblob.getProperties().getRehydrationStatus()); } catch (Exception e) { // Output the stack trace. e.printStackTrace(); } }
private void copyBlob(CloudBlob blob, CloudBlobDirectory newParent) throws IOException { checkArgument(blob instanceof CloudBlockBlob, "Only page blobs are supported for the rename"); try { String blobName = getName(blob); CloudBlockBlob newBlob = newParent.getBlockBlobReference(blobName); newBlob.startCopy(blob.getUri()); boolean isStatusPending = true; while (isStatusPending) { newBlob.downloadAttributes(); if (newBlob.getCopyState().getStatus() == CopyStatus.PENDING) { Thread.sleep(100); } else { isStatusPending = false; } } CopyStatus finalStatus = newBlob.getCopyState().getStatus(); if (newBlob.getCopyState().getStatus() != CopyStatus.SUCCESS) { throw new IOException("Invalid copy status for " + blob.getUri().getPath() + ": " + finalStatus); } } catch (StorageException | InterruptedException | URISyntaxException e) { throw new IOException(e); } }
private void copyBlob(CloudBlob blob, CloudBlobDirectory newParent) throws IOException { checkArgument(blob instanceof CloudBlockBlob, "Only page blobs are supported for the rename"); try { String blobName = getName(blob); CloudBlockBlob newBlob = newParent.getBlockBlobReference(blobName); newBlob.startCopy(blob.getUri()); boolean isStatusPending = true; while (isStatusPending) { newBlob.downloadAttributes(); if (newBlob.getCopyState().getStatus() == CopyStatus.PENDING) { Thread.sleep(100); } else { isStatusPending = false; } } CopyStatus finalStatus = newBlob.getCopyState().getStatus(); if (newBlob.getCopyState().getStatus() != CopyStatus.SUCCESS) { throw new IOException("Invalid copy status for " + blob.getUri().getPath() + ": " + finalStatus); } } catch (StorageException | InterruptedException | URISyntaxException e) { throw new IOException(e); } }
@Override String getStateDebug() { String retval = "uninitialized"; try { this.blob.downloadAttributes(); BlobProperties props = this.blob.getProperties(); retval = props.getLeaseState().toString() + " " + props.getLeaseStatus().toString() + " " + props.getLeaseDuration().toString(); } catch (StorageException e) { retval = "downloadAttributes on the blob caught " + e.toString(); } return retval; } }
@Override public boolean blockRestored(String id) { try { CloudBlockBlob blob = container.getBlockBlobReference("blocks/" + id); if(!blob.exists()) return true; blob.downloadAttributes(); if (blob.getProperties().getStandardBlobTier().equals(StandardBlobTier.HOT)) { return true; } else { if (blob.getProperties().getRehydrationStatus() == null || blob.getProperties().getRehydrationStatus().equals(RehydrationStatus.UNKNOWN)) { SDFSLogger.getLog().warn("rehydration status unknow for " + id + " will attempt to rehydrate"); blob.uploadStandardBlobTier(StandardBlobTier.HOT); } return false; } } catch (Exception e) { SDFSLogger.getLog().warn("error while checking block [" + id + "] restored", e); return false; } }
@Test public void testEmpty() throws URISyntaxException, StorageException, IOException { String blobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testblob"); CloudBlockBlob blockBlob = this.container.getBlockBlobReference(blobName); BlobOutputStream str = blockBlob.openOutputStream(); str.close(); CloudBlockBlob blockBlob2 = this.container.getBlockBlobReference(blobName); blockBlob2.downloadAttributes(); assertEquals(0, blockBlob2.getProperties().getLength()); ArrayList<BlockEntry> blocks = blockBlob2.downloadBlockList(BlockListingFilter.ALL, null, null, null); assertEquals(0, blocks.size()); }
/** * @throws StorageException * @throws URISyntaxException * @throws IOException * @throws InterruptedException */ @Test @Category({ DevFabricTests.class, DevStoreTests.class, SlowTests.class }) public void testBlobNamePlusEncodingTest() throws StorageException, URISyntaxException, IOException, InterruptedException { final int length = 1 * 1024; final CloudBlockBlob originalBlob = (CloudBlockBlob) BlobTestHelper.uploadNewBlob(this.container, BlobType.BLOCK_BLOB, "a+b.txt", length, null); final CloudBlockBlob copyBlob = this.container.getBlockBlobReference(originalBlob.getName() + "copyed"); copyBlob.startCopy(originalBlob); BlobTestHelper.waitForCopy(copyBlob); copyBlob.downloadAttributes(); }
@Test public void testBlobAttributesEncryption() throws URISyntaxException, StorageException, IOException { this.blob.downloadAttributes(); assertTrue(this.blob.getProperties().isServerEncrypted()); CloudBlockBlob testBlob = this.container.getBlockBlobReference(this.blob.getName()); testBlob.downloadText(); assertTrue(testBlob.getProperties().isServerEncrypted()); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testUploadDownloadBlobProperties() throws URISyntaxException, StorageException, IOException { final int length = 0; // do this to make sure the set MD5 can be compared without an exception being thrown BlobRequestOptions options = new BlobRequestOptions(); options.setDisableContentMD5Validation(true); // with explicit upload/download of properties String blockBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); CloudBlockBlob blockBlobRef1 = this.container.getBlockBlobReference(blockBlobName1); blockBlobRef1.upload(BlobTestHelper.getRandomDataStream(length), length); // this is not set by upload (it is for page blob!), so set this manually blockBlobRef1.getProperties().setLength(length); BlobTestHelper.setBlobProperties(blockBlobRef1); BlobProperties props1 = blockBlobRef1.getProperties(); blockBlobRef1.uploadProperties(); blockBlobRef1.downloadAttributes(null, options, null); BlobProperties props2 = blockBlobRef1.getProperties(); Assert.assertEquals(props1.getLength(), props2.getLength()); BlobTestHelper.assertAreEqual(props1, props2); // by uploading/downloading the blob blockBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); blockBlobRef1 = this.container.getBlockBlobReference(blockBlobName1); BlobTestHelper.setBlobProperties(blockBlobRef1); props1 = blockBlobRef1.getProperties(); blockBlobRef1.upload(BlobTestHelper.getRandomDataStream(length), length); blockBlobRef1.download(new ByteArrayOutputStream(), null, options, null); props2 = blockBlobRef1.getProperties(); BlobTestHelper.assertAreEqual(props1, props2); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testCloudBlockBlobUploadStandardTier() throws StorageException, IOException, URISyntaxException { for (StandardBlobTier standardBlobTier : StandardBlobTier.values()) { if (standardBlobTier == StandardBlobTier.UNKNOWN) { continue; } final String blobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlob"); final CloudBlockBlob blob = this.container.getBlockBlobReference(blobName); blob.uploadText("text"); blob.uploadStandardBlobTier(standardBlobTier); assertEquals(standardBlobTier, blob.getProperties().getStandardBlobTier()); assertNull(blob.getProperties().getPremiumPageBlobTier()); assertNull(blob.getProperties().getRehydrationStatus()); CloudBlockBlob blob2 = this.container.getBlockBlobReference(blobName); blob2.downloadAttributes(); assertEquals(standardBlobTier, blob2.getProperties().getStandardBlobTier()); assertNull(blob2.getProperties().getPremiumPageBlobTier()); assertNull(blob2.getProperties().getRehydrationStatus()); CloudBlockBlob blob3 = (CloudBlockBlob)this.container.listBlobs().iterator().next(); assertEquals(standardBlobTier, blob3.getProperties().getStandardBlobTier()); assertNull(blob3.getProperties().getPremiumPageBlobTier()); assertNull(blob3.getProperties().getRehydrationStatus()); blob.deleteIfExists(); } }