@Override public void run() { try { CloudBlobContainer container = blobClient.getContainerReference(azureContainer); container.createIfNotExists(); CloudBlockBlob blob = container.getBlockBlobReference(azureKey); blob.upload(new java.io.FileInputStream(localFile), localFile.length()); } catch (URISyntaxException e) { throw new RuntimeException(e); } catch (StorageException e) { throw new RuntimeException(e); } catch (FileNotFoundException e) { throw new RuntimeException(e); } catch (IOException e) { throw new RuntimeException(e); } } });
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null); CloudBlobContainer container = blobClient.getContainerReference(containerName); for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, operationContext)) { if (blob instanceof CloudBlob) { CloudBlob cloudBlob = (CloudBlob) blob; .primaryUri(uri.getPrimaryUri().toString()) .blobName(cloudBlob.getName()) .containerName(containerName) builder.secondaryUri(uri.getSecondaryUri().toString());
@Override protected void write(final List<PingData> list, final String clustername) { if (list == null || clustername == null) { return; } String filename = addressToFilename(clustername, local_addr); ByteArrayOutputStream out = new ByteArrayOutputStream(STREAM_BUFFER_SIZE); try { write(list, out); byte[] data = out.toByteArray(); // Upload the file CloudBlockBlob blob = containerReference.getBlockBlobReference(filename); blob.upload(new ByteArrayInputStream(data), data.length); } catch (Exception ex) { log.error("Error marshalling and uploading ping data.", ex); } }
@Test public void testPageBlobDownloadRangeTest() throws URISyntaxException, StorageException, IOException { byte[] buffer = BlobTestHelper.getRandomBuffer(2 * 1024); CloudPageBlob blob = this.container.getPageBlobReference("blob1"); ByteArrayInputStream wholeBlob = new ByteArrayInputStream(buffer); blob.upload(wholeBlob, 2 * 1024); ByteArrayOutputStream blobStream = new ByteArrayOutputStream(); try { blob.downloadRange(0, new Long(0), blobStream); } catch (IndexOutOfBoundsException ex) { } blob.downloadRange(0, new Long(1024), blobStream); assertEquals(blobStream.size(), 1024); BlobTestHelper.assertStreamsAreEqualAtIndex(new ByteArrayInputStream(blobStream.toByteArray()), wholeBlob, 0, 0, 1024, 2 * 1024); CloudPageBlob blob2 = this.container.getPageBlobReference("blob1"); try { blob.downloadRange(1024, new Long(0), blobStream); } catch (IndexOutOfBoundsException ex) { } ByteArrayOutputStream blobStream2 = new ByteArrayOutputStream(); blob2.downloadRange(1024, new Long(1024), blobStream2); BlobTestHelper.assertStreamsAreEqualAtIndex(new ByteArrayInputStream(blobStream2.toByteArray()), wholeBlob, 1024, 1024, 1024, 2 * 1024); BlobTestHelper.assertAreEqual(blob, blob2); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testBlockBlobDownloadRangeTest() throws URISyntaxException, StorageException, IOException { byte[] buffer = BlobTestHelper.getRandomBuffer(2 * 1024); CloudBlockBlob blob = this.container.getBlockBlobReference("blob1"); ByteArrayInputStream wholeBlob = new ByteArrayInputStream(buffer); blob.upload(wholeBlob, -1); ByteArrayOutputStream blobStream = new ByteArrayOutputStream(); try { blob.downloadRange(0, new Long(0), blobStream, null, null, null); } catch (IndexOutOfBoundsException ex) { } blob.downloadRange(0, new Long(1024), blobStream); assertEquals(blobStream.size(), 1024); BlobTestHelper.assertStreamsAreEqualAtIndex(new ByteArrayInputStream(blobStream.toByteArray()), wholeBlob, 0, 0, 1024, 2 * 1024); CloudBlockBlob blob2 = this.container.getBlockBlobReference("blob1"); try { blob.downloadRange(1024, new Long(0), blobStream, null, null, null); } catch (IndexOutOfBoundsException ex) { } ByteArrayOutputStream blobStream2 = new ByteArrayOutputStream(); blob2.downloadRange(1024, new Long(1024), blobStream2); BlobTestHelper.assertStreamsAreEqualAtIndex(new ByteArrayInputStream(blobStream2.toByteArray()), wholeBlob, 1024, 1024, 1024, 2 * 1024); blob2.downloadAttributes(); BlobTestHelper.assertAreEqual(blob, blob2); }
@Test @Category(SlowTests.class) public void testCopyPageBlobSas() throws Exception { // Create source on server. final CloudBlobContainer container = BlobTestHelper.getRandomContainerReference(); try { container.create(); final CloudPageBlob source = container.getPageBlobReference("source"); source.getMetadata().put("Test", "value"); final int length = 512; final ByteArrayInputStream data = BlobTestHelper.getRandomDataStream(length); source.upload(data, length); final CloudFile destination = doCloudBlobCopy(source, length); final ByteArrayOutputStream copyData = new ByteArrayOutputStream(); destination.download(copyData); BlobTestHelper.assertStreamsAreEqual(data, new ByteArrayInputStream(copyData.toByteArray())); } finally { container.deleteIfExists(); } }
@Test @Category(SlowTests.class) public void testBlobSaS() throws InvalidKeyException, IllegalArgumentException, StorageException, URISyntaxException, InterruptedException { SharedAccessBlobPolicy sp = createSharedAccessPolicy( EnumSet.of(SharedAccessBlobPermissions.READ, SharedAccessBlobPermissions.LIST), 3600); BlobContainerPermissions perms = new BlobContainerPermissions(); perms.getSharedAccessPolicies().put("readperm", sp); this.container.uploadPermissions(perms); Thread.sleep(30000); CloudBlockBlob sasBlob = new CloudBlockBlob(new URI(this.blob.getUri().toString() + "?" + this.blob.generateSharedAccessSignature(null, "readperm"))); sasBlob.download(new ByteArrayOutputStream()); // do not give the client and check that the new blob's client has the correct perms CloudBlob blobFromUri = new CloudBlockBlob(PathUtility.addToQuery(this.blob.getStorageUri(), this.blob.generateSharedAccessSignature(null, "readperm"))); assertEquals(StorageCredentialsSharedAccessSignature.class.toString(), blobFromUri.getServiceClient() .getCredentials().getClass().toString()); // create credentials from sas StorageCredentials creds = new StorageCredentialsSharedAccessSignature( this.blob.generateSharedAccessSignature(null, "readperm")); CloudBlobClient bClient = new CloudBlobClient(sasBlob.getServiceClient().getStorageUri(), creds); CloudBlockBlob blobFromClient = bClient.getContainerReference(this.blob.getContainer().getName()) .getBlockBlobReference(this.blob.getName()); assertEquals(StorageCredentialsSharedAccessSignature.class.toString(), blobFromClient.getServiceClient() .getCredentials().getClass().toString()); assertEquals(bClient, blobFromClient.getServiceClient()); }
@Test public void testWithAccessCondition() throws URISyntaxException, StorageException, IOException { int blobLengthToUse = 8 * 512; byte[] buffer = BlobTestHelper.getRandomBuffer(blobLengthToUse); String blobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testblob"); AccessCondition accessCondition = AccessCondition.generateIfNotModifiedSinceCondition(new Date()); CloudBlockBlob blockBlob = this.container.getBlockBlobReference(blobName); BlobOutputStream blobOutputStream = blockBlob.openOutputStream(accessCondition, null, null); ByteArrayInputStream inputStream = new ByteArrayInputStream(buffer); blobOutputStream.write(inputStream, 512); inputStream = new ByteArrayInputStream(buffer, 512, 3 * 512); blobOutputStream.write(inputStream, 3 * 512); blobOutputStream.close(); byte[] result = new byte[blobLengthToUse]; blockBlob.downloadToByteArray(result, 0); int i = 0; for (; i < 4 * 512; i++) { assertEquals(buffer[i], result[i]); } for (; i < 8 * 512; i++) { assertEquals(0, result[i]); } }
this.container.uploadPermissions(perms); Thread.sleep(30000); String containerReadListSas = this.container.generateSharedAccessSignature(sp2, null); CloudBlobContainer readListContainer = new CloudBlobContainer(PathUtility.addToQuery(this.container.getUri(), containerReadListSas)); assertEquals(StorageCredentialsSharedAccessSignature.class.toString(), readListContainer.getServiceClient() .getCredentials().getClass().toString()); CloudBlockBlob blobFromSasContainer = readListContainer.getBlockBlobReference(this.blob.getName()); blobFromSasContainer.download(new ByteArrayOutputStream()); CloudBlobContainer containerFromUri = new CloudBlobContainer(PathUtility.addToQuery( readListContainer.getStorageUri(), this.container.generateSharedAccessSignature(null, "readlist"))); assertEquals(StorageCredentialsSharedAccessSignature.class.toString(), containerFromUri.getServiceClient() .getCredentials().getClass().toString()); this.container.generateSharedAccessSignature(null, "readlist")); CloudBlobClient bClient = new CloudBlobClient(this.container.getServiceClient().getStorageUri(), creds); CloudBlobContainer containerFromClient = bClient.getContainerReference(this.container.getName()); assertEquals(StorageCredentialsSharedAccessSignature.class.toString(), containerFromClient.getServiceClient() .getCredentials().getClass().toString()); assertEquals(bClient, containerFromClient.getServiceClient());
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testVerifyTransactionalMD5ValidationMissingOverallMD5() throws URISyntaxException, StorageException, IOException { final String blockBlobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef = this.container.getBlockBlobReference(blockBlobName); final int length = 2 * 1024 * 1024; ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length); BlobRequestOptions options = new BlobRequestOptions(); options.setSingleBlobPutThresholdInBytes(1024*1024); options.setDisableContentMD5Validation(true); options.setStoreBlobContentMD5(false); blockBlobRef.upload(srcStream, -1, null, options, null); options.setDisableContentMD5Validation(false); options.setStoreBlobContentMD5(true); options.setUseTransactionalContentMD5(true); final CloudBlockBlob blockBlobRef2 = this.container.getBlockBlobReference(blockBlobName); blockBlobRef2.downloadRange(1024, (long)1024, new ByteArrayOutputStream(), null, options, null); assertNull(blockBlobRef2.getProperties().getContentMD5()); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testBlobEmptyHeaderSigningTest() throws URISyntaxException, StorageException, IOException { final String blockBlobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef = this.container.getBlockBlobReference(blockBlobName); final int length = 2 * 1024; ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length); OperationContext context = new OperationContext(); context.getSendingRequestEventHandler().addListener(new StorageEvent<SendingRequestEvent>() { @Override public void eventOccurred(SendingRequestEvent eventArg) { HttpURLConnection connection = (HttpURLConnection) eventArg.getConnectionObject(); connection.setRequestProperty("x-ms-foo", ""); } }); blockBlobRef.upload(srcStream, -1, null, null, context); blockBlobRef.download(new ByteArrayOutputStream(), null, null, context); }
public void UploadtoAzureStorage(InputStream fileInputStream,String filename) { try { CloudStorageAccount storageAccount = CloudStorageAccount.parse(storageConnectionString); CloudBlobClient blobClient = storageAccount.createCloudBlobClient(); CloudBlobContainer container = blobClient.getContainerReference("mycontainer"); byte[] bytes = IOUtils.toByteArray(fileInputStream); CloudBlockBlob blob = container.getBlockBlobReference(filename); BlobOutputStream blobOutputStream = blob.openOutputStream(); ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes); int next = inputStream.read(); while (next != -1) { blobOutputStream.write(next); next = inputStream.read(); } blobOutputStream.close(); } catch (Exception e) { e.printStackTrace(); } }
@Test public void testBlobUriOnlyConstructors() throws URISyntaxException, StorageException, InvalidKeyException { URI blobURI = new URI(container.getUri().toString() + "/anonblob"); CloudBlockBlob blob = new CloudBlockBlob(blobURI); assertEquals("anonblob", blob.getName()); assertNotNull(blob.getServiceClient()); assertEquals(StorageCredentialsAnonymous.class, blob.getServiceClient().getCredentials().getClass()); blob = container.getBlockBlobReference("anonblob"); String sas = blob.generateSharedAccessSignature(null, "dummyPolicy"); blobURI = new URI(container.getUri().toString() + "/anonblob?" + sas); blob = new CloudBlockBlob(blobURI); assertEquals("anonblob", blob.getName()); assertNotNull(blob.getServiceClient()); assertEquals(StorageCredentialsSharedAccessSignature.class, blob.getServiceClient().getCredentials().getClass()); }
@NotNull public BlobFile createBlobFile(@NotNull ClientStorageAccount storageAccount, @NotNull BlobDirectory parentBlobDirectory, @NotNull BlobFile blobFile) throws AzureCmdException { try { CloudBlobClient client = getCloudBlobClient(storageAccount); String containerName = parentBlobDirectory.getContainerName(); CloudBlobContainer container = client.getContainerReference(containerName); CloudBlobDirectory parentDirectory = container.getDirectoryReference(parentBlobDirectory.getPath()); CloudBlob blob = getCloudBlob(parentDirectory, blobFile); blob.upload(new ByteArrayInputStream(new byte[0]), 0); return reloadBlob(blob, containerName, blobFile); } catch (Throwable t) { throw new AzureCmdException("Error creating the Blob File", t); } }
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile); CloudBlobContainer container = blobClient.getContainerReference(containerName); CloudBlob blob = container.getBlockBlobReference(blobPath); BlobProperties properties = blob.getProperties(); attributes.put("azure.container", containerName); attributes.put("azure.primaryUri", blob.getSnapshotQualifiedUri().toString()); attributes.put("azure.etag", properties.getEtag()); attributes.put("azure.length", String.valueOf(length)); session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
/** * Download file from Azure and save to target path specified. Return true if download succeeds * @param filePathName * @return */ private static Boolean downloadFromAzure(@NotNull Path filePathName) { boolean result = true; try { URI cloudBlobUri = new URI(AZURE_BLOB_URI); CloudBlobClient serviceClient = new CloudBlobClient(cloudBlobUri); CloudBlobContainer container = serviceClient.getContainerReference("libcontainer1"); CloudBlockBlob blob = container.getBlockBlobReference(filePathName.getFileName().toString()); File downloadingFile = filePathName.toFile(); blob.downloadToFile(downloadingFile.getAbsolutePath()); } catch (Exception e) { result = false; log.warning("Fail to download file from Azure: " + e.getMessage()); } return result; }
public static CloudBlobDirectory cloudBlobDirectoryFrom(StorageCredentials credentials, String uri, String dir) throws URISyntaxException, StorageException { StorageUri storageUri = new StorageUri(new URI(uri)); CloudBlobContainer container = new CloudBlobContainer(storageUri, credentials); return container.getDirectoryReference(dir); }
public static String uploadFileAsBlob(final File fileToUpload, final CloudStorageAccount storageAccount, final String containerName, final String blobName) throws Exception { final CloudBlobContainer blobContainer = getBlobContainer(storageAccount, containerName); blobContainer.createIfNotExists(BlobContainerPublicAccessType.BLOB, null, null); final CloudBlockBlob blob = blobContainer.getBlockBlobReference(blobName); blob.upload(new FileInputStream(fileToUpload), fileToUpload.length()); return blob.getUri().toString(); }
@NotNull public BlobDirectory getRootDirectory(@NotNull String connectionString, @NotNull BlobContainer blobContainer) throws AzureCmdException { try { CloudBlobClient client = getCloudBlobClient(connectionString); CloudBlobContainer container = client.getContainerReference(blobContainer.getName()); CloudBlobDirectory directory = container.getDirectoryReference(""); String uri = directory.getUri() != null ? directory.getUri().toString() : ""; return new BlobDirectory("", uri, blobContainer.getName(), ""); } catch (Throwable t) { throw new AzureCmdException("Error retrieving the root Blob Directory", t); } }
@NotNull public static String listBlobs(@NotNull final StorageInputs inputs) throws Exception { final CloudBlobClient blobClient = getCloudBlobClient(inputs); final CloudBlobContainer container = blobClient.getContainerReference(inputs.getContainerName()); final List<String> blobList = new ArrayList<>(); for (final ListBlobItem blobItem : container.listBlobs()) { final String path = blobItem.getUri().getPath(); blobList.add(path.substring(path.lastIndexOf(FORWARD_SLASH) + 1)); } return StringUtilities.join(blobList, COMMA); }