try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile); CloudBlobContainer container = blobClient.getContainerReference(containerName); CloudBlob blob = container.getBlockBlobReference(blobPath); blob.upload(in, length, null, null, operationContext); BlobProperties properties = blob.getProperties(); attributes.put("azure.container", containerName); attributes.put("azure.primaryUri", blob.getSnapshotQualifiedUri().toString()); attributes.put("azure.etag", properties.getEtag()); attributes.put("azure.length", String.valueOf(length)); session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
String storageConnectionString = String.format(AzureStorageUtils.FORMAT_BASE_URI, accountName); StorageCredentials creds = new StorageCredentialsSharedAccessSignature(sasToken); cloudBlobClient = new CloudBlobClient(new URI(storageConnectionString), creds); } else { String blobConnString = String.format(AzureStorageUtils.FORMAT_BLOB_CONNECTION_STRING, accountName, accountKey); CloudStorageAccount storageAccount = CloudStorageAccount.parse(blobConnString); cloudBlobClient = storageAccount.createCloudBlobClient();
@Test @Category(SlowTests.class) public void testBlobSaS() throws InvalidKeyException, IllegalArgumentException, StorageException, URISyntaxException, InterruptedException { SharedAccessBlobPolicy sp = createSharedAccessPolicy( EnumSet.of(SharedAccessBlobPermissions.READ, SharedAccessBlobPermissions.LIST), 3600); BlobContainerPermissions perms = new BlobContainerPermissions(); perms.getSharedAccessPolicies().put("readperm", sp); this.container.uploadPermissions(perms); Thread.sleep(30000); CloudBlockBlob sasBlob = new CloudBlockBlob(new URI(this.blob.getUri().toString() + "?" + this.blob.generateSharedAccessSignature(null, "readperm"))); sasBlob.download(new ByteArrayOutputStream()); // do not give the client and check that the new blob's client has the correct perms CloudBlob blobFromUri = new CloudBlockBlob(PathUtility.addToQuery(this.blob.getStorageUri(), this.blob.generateSharedAccessSignature(null, "readperm"))); assertEquals(StorageCredentialsSharedAccessSignature.class.toString(), blobFromUri.getServiceClient() .getCredentials().getClass().toString()); // create credentials from sas StorageCredentials creds = new StorageCredentialsSharedAccessSignature( this.blob.generateSharedAccessSignature(null, "readperm")); CloudBlobClient bClient = new CloudBlobClient(sasBlob.getServiceClient().getStorageUri(), creds); CloudBlockBlob blobFromClient = bClient.getContainerReference(this.blob.getContainer().getName()) .getBlockBlobReference(this.blob.getName()); assertEquals(StorageCredentialsSharedAccessSignature.class.toString(), blobFromClient.getServiceClient() .getCredentials().getClass().toString()); assertEquals(bClient, blobFromClient.getServiceClient()); }
@Test public void testBlobUriOnlyConstructors() throws URISyntaxException, StorageException, InvalidKeyException { URI blobURI = new URI(container.getUri().toString() + "/anonblob"); CloudBlockBlob blob = new CloudBlockBlob(blobURI); assertEquals("anonblob", blob.getName()); assertNotNull(blob.getServiceClient()); assertEquals(StorageCredentialsAnonymous.class, blob.getServiceClient().getCredentials().getClass()); blob = container.getBlockBlobReference("anonblob"); String sas = blob.generateSharedAccessSignature(null, "dummyPolicy"); blobURI = new URI(container.getUri().toString() + "/anonblob?" + sas); blob = new CloudBlockBlob(blobURI); assertEquals("anonblob", blob.getName()); assertNotNull(blob.getServiceClient()); assertEquals(StorageCredentialsSharedAccessSignature.class, blob.getServiceClient().getCredentials().getClass()); }
@Override public Path copy(final Path source, final Path copy, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { final CloudBlob target = session.getClient().getContainerReference(containerService.getContainer(copy).getName()) .getAppendBlobReference(containerService.getKey(copy)); final CloudBlob blob = session.getClient().getContainerReference(containerService.getContainer(source).getName()) .getBlobReferenceFromServer(containerService.getKey(source)); final BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(PreferencesFactory.get().getBoolean("azure.upload.md5")); final URI s = session.getHost().getCredentials().isTokenAuthentication() ? URI.create(blob.getUri().toString() + session.getHost().getCredentials().getToken()) : blob.getUri(); final String id = target.startCopy(s, AccessCondition.generateEmptyCondition(), AccessCondition.generateEmptyCondition(), options, context); if(log.isDebugEnabled()) { log.debug(String.format("Started copy for %s with copy operation ID %s", copy, id)); } // Copy original file attributes return new Path(copy.getParent(), copy.getName(), copy.getType(), new PathAttributes(source.attributes())); } catch(StorageException e) { throw new AzureExceptionMappingService().map("Cannot copy {0}", e, source); } catch(URISyntaxException e) { throw new NotfoundException(e.getMessage(), e); } }
public static String getBlobSasUri(String blobLink, String saKey) throws URISyntaxException, StorageException, InvalidKeyException, MalformedURLException { CloudStorageAccount cloudStorageAccount = getCloudStorageAccount(blobLink, saKey); CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient(); String path = new URI(blobLink).getPath(); if (path == null) { throw new IllegalArgumentException("Invalid blobLink: " + blobLink); CloudBlobContainer container = blobClient.getContainerReference(containerName); sharedAccessBlobPolicy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ)); BlobContainerPermissions containerPermissions = new BlobContainerPermissions(); container.uploadPermissions(containerPermissions); String signature = container.generateSharedAccessSignature(sharedAccessBlobPolicy, null); return blobLink + "?" + signature;
@NotNull public BlobDirectory getRootDirectory(@NotNull String connectionString, @NotNull BlobContainer blobContainer) throws AzureCmdException { try { CloudBlobClient client = getCloudBlobClient(connectionString); CloudBlobContainer container = client.getContainerReference(blobContainer.getName()); CloudBlobDirectory directory = container.getDirectoryReference(""); String uri = directory.getUri() != null ? directory.getUri().toString() : ""; return new BlobDirectory("", uri, blobContainer.getName(), ""); } catch (Throwable t) { throw new AzureCmdException("Error retrieving the root Blob Directory", t); } }
/** * Download file from Azure and save to target path specified. Return true if download succeeds * @param filePathName * @return */ private static Boolean downloadFromAzure(@NotNull Path filePathName) { boolean result = true; try { URI cloudBlobUri = new URI(AZURE_BLOB_URI); CloudBlobClient serviceClient = new CloudBlobClient(cloudBlobUri); CloudBlobContainer container = serviceClient.getContainerReference("libcontainer1"); CloudBlockBlob blob = container.getBlockBlobReference(filePathName.getFileName().toString()); File downloadingFile = filePathName.toFile(); blob.downloadToFile(downloadingFile.getAbsolutePath()); } catch (Exception e) { result = false; log.warning("Fail to download file from Azure: " + e.getMessage()); } return result; }
public class SASmeta { public static void main(String[] args) throws URISyntaxException, FileNotFoundException, StorageException, IOException { URI baseuri = new URI("http://grassy.blob.core.windows.net"); CloudBlobClient blobclient = new CloudBlobClient(baseuri); MyUploadBlob("container1","sr=c&sv=2012-02-12&sig=b%2BK%2FmX8r7dLCVxs5PSPmmji1L6kSxNupp9SKF7yj27w%3D&si=r",blobclient); } public static void MyUploadBlob(String containerName, String containerSAS, CloudBlobClient blobClient) throws URISyntaxException, StorageException, FileNotFoundException, IOException { String blobName = "image1.jpg"; String localFileName = "c:\\myimages\\image1.jpg"; URI uri = new URI(blobClient.getEndpoint().toString() + "/" + containerName + "/" + blobName + "?" + containerSAS); CloudBlockBlob sasBlob = new CloudBlockBlob(uri, blobClient); HashMap<String, String> user = new HashMap<String, String>(); user.put("firstname", "Joe"); user.put("lastname", "Smith" ); user.put("age", "28"); user.put("presenter", "no"); sasBlob.setMetadata(user); File fileReference = new File(localFileName); sasBlob.upload(new FileInputStream(fileReference), fileReference.length()); System.out.println("The blob: " + blobName + " has been uploaded to:"); System.out.println(uri); } }
FileNotFoundException, StorageException, IOException URI baseuri = new URI("http://grassy.blob.core.windows.net"); CloudBlobClient blobclient = new CloudBlobClient(baseuri); MyDownloadBlob("container1", "sr=c&sv=2012-02-12&sig=nnPn5P5nnPPnn5Pnn5PPnPPPnPPP5PPPPPP%5PPnn5PPn%55&si=heath", URI uri = new URI(blobClient.getEndpoint().toString() + "/" + containerName + "/" + blobName + "?" + containerSAS); CloudBlockBlob sasBlob = new CloudBlockBlob(uri, blobClient); File fileTarget = new File(localFileName); sasBlob.download(new FileOutputStream(fileTarget)); HashMap<String, String> user = new HashMap<String, String>(); user = sasBlob.getMetadata(); String name = (user.get("firstname") + " " + user.get("lastname")); String age = ("age: " + user.get("age"));
URI BlobEndPoint = new URI("http://127.0.0.1:10000/devstoreaccount1"); CloudBlobClient bClient = new CloudBlobClient(BlobEndPoint, new StorageCredentialsAccountAndKey(AccountName, AccountSecurityKey));
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile); CloudBlobContainer container = blobClient.getContainerReference(containerName); final CloudBlob blob = container.getBlockBlobReference(blobPath); blob.download(os, null, null, operationContext); } catch (StorageException e) { storedException.set(e); long length = blob.getProperties().getLength(); attributes.put("azure.length", String.valueOf(length)); session.getProvenanceReporter().fetch(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis); } catch (IllegalArgumentException | URISyntaxException | StorageException | ProcessException e) { if (e instanceof ProcessException && storedException.get() == null) {
CloudBlobContainer container = blobClient.getContainerReference(containerName); CloudBlob blob = container.getBlockBlobReference(blobPath); blob.deleteIfExists(DeleteSnapshotsOption.valueOf(deleteSnapshotOptions), null, null, operationContext); session.transfer(flowFile, REL_SUCCESS); session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis); } catch ( StorageException | URISyntaxException e) { getLogger().error("Failed to delete the specified blob {} from Azure Storage. Routing to failure", new Object[]{blobPath}, e);
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null); CloudBlobContainer container = blobClient.getContainerReference(containerName); for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, operationContext)) { if (blob instanceof CloudBlob) { CloudBlob cloudBlob = (CloudBlob) blob; BlobProperties properties = cloudBlob.getProperties(); StorageUri uri = cloudBlob.getSnapshotQualifiedStorageUri(); .primaryUri(uri.getPrimaryUri().toString()) .blobName(cloudBlob.getName()) .containerName(containerName) .contentType(properties.getContentType()) builder.secondaryUri(uri.getSecondaryUri().toString());
@Test public void testCloudStorageAccountClientUriVerify() throws URISyntaxException, StorageException { StorageCredentialsAccountAndKey cred = new StorageCredentialsAccountAndKey(ACCOUNT_NAME, ACCOUNT_KEY); CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(cred, true); CloudBlobClient blobClient = cloudStorageAccount.createCloudBlobClient(); CloudBlobContainer container = blobClient.getContainerReference("container1"); assertEquals(cloudStorageAccount.getBlobEndpoint().toString() + "/container1", container.getUri().toString()); CloudQueueClient queueClient = cloudStorageAccount.createCloudQueueClient(); CloudQueue queue = queueClient.getQueueReference("queue1"); assertEquals(cloudStorageAccount.getQueueEndpoint().toString() + "/queue1", queue.getUri().toString()); CloudTableClient tableClient = cloudStorageAccount.createCloudTableClient(); CloudTable table = tableClient.getTableReference("table1"); assertEquals(cloudStorageAccount.getTableEndpoint().toString() + "/table1", table.getUri().toString()); CloudFileClient fileClient = cloudStorageAccount.createCloudFileClient(); CloudFileShare share = fileClient.getShareReference("share1"); assertEquals(cloudStorageAccount.getFileEndpoint().toString() + "/share1", share.getUri().toString()); }
try { File source = new File(filePath); // File path String extantion = source.getAbsolutePath().substring(source.getAbsolutePath().lastIndexOf(".")); // create unique number to identify the image/file. // you can also specify some name to image/file String uniqueID = "image_"+ UUID.randomUUID().toString().replace("-", "")+extantion; String blobUri = MY_URL_TO_UPLOAD_PROFILE_IMAGE + sas.replaceAll("\"",""); StorageUri storage = new StorageUri(URI.create(blobUri)); CloudBlobClient blobCLient = new CloudBlobClient(storage); CloudBlobContainer container = blobCLient.getContainerReference(""); CloudBlockBlob blob = container.getBlockBlobReference(uniqueID); BlobOutputStream blobOutputStream = blob.openOutputStream(); byte[] buffer = fileToByteConverter(source); ByteArrayInputStream inputStream = new ByteArrayInputStream(buffer); int next = inputStream.read(); while (next != -1) { blobOutputStream.write(next); next = inputStream.read(); } blobOutputStream.close(); // YOUR IMAGE/FILE GET UPLOADED HERE // IF YOU HAVE FOLLOW DOCUMENT, YOU WILL RECEIVE IMAGE/FILE URL HERE } catch (StorageException e) { e.printStackTrace(); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (Exception e){ e.printStackTrace(); }
private DescriptiveUrl createSignedUrl(final Path file, int seconds) { final CloudBlob blob; try { if(!session.isConnected()) { return DescriptiveUrl.EMPTY; } blob = session.getClient().getContainerReference(containerService.getContainer(file).getName()) .getBlobReferenceFromServer(containerService.getKey(file)); } catch(URISyntaxException | StorageException e) { return DescriptiveUrl.EMPTY; } final String token; try { token = blob.generateSharedAccessSignature(this.getPolicy(seconds), null); } catch(InvalidKeyException | StorageException e) { return DescriptiveUrl.EMPTY; } return new DescriptiveUrl(URI.create(String.format("%s://%s%s?%s", Scheme.https.name(), session.getHost().getHostname(), URIEncoder.encode(file.getAbsolute()), token)), DescriptiveUrl.Type.signed, MessageFormat.format(LocaleFactory.localizedString("{0} URL"), LocaleFactory.localizedString("Pre-Signed", "S3")) + " (" + MessageFormat.format(LocaleFactory.localizedString("Expires {0}", "S3") + ")", UserDateFormatterFactory.get().getShortFormat(this.getExpiry(seconds)))); }
@NotNull public static String listBlobs(@NotNull final StorageInputs inputs) throws Exception { final CloudBlobClient blobClient = getCloudBlobClient(inputs); final CloudBlobContainer container = blobClient.getContainerReference(inputs.getContainerName()); final List<String> blobList = new ArrayList<>(); for (final ListBlobItem blobItem : container.listBlobs()) { final String path = blobItem.getUri().getPath(); blobList.add(path.substring(path.lastIndexOf(FORWARD_SLASH) + 1)); } return StringUtilities.join(blobList, COMMA); }
storageConnectionString = storageConnectionString + "EndpointSuffix=" + this.endpoint; account = CloudStorageAccount.parse(storageConnectionString); serviceClient = account.createCloudBlobClient(); serviceClient.getDefaultRequestOptions().setConcurrentRequestCount(Main.dseIOThreads * 2); if (tier != null && (tier.equals(StandardBlobTier.ARCHIVE) || tier.equals(StandardBlobTier.COOL))) { this.bio = new BlobDataIO(this.name + "table", this.accessKey, this.secretKey, connectionProtocol); container = serviceClient.getContainerReference(this.name); container.createIfNotExists(null, null, opContext); container.downloadAttributes(); HashMap<String, String> md = container.getMetadata(); if (md.size() == 0) this.clustered = true; if (this.standAlone) { CloudBlockBlob blob = container.getBlockBlobReference(lbi); if (blob.exists()) { blob.downloadAttributes(); HashMap<String, String> metaData = blob.getMetadata(); if (metaData.containsKey("currentlength")) { sz = Long.parseLong(metaData.get("currentlength"));
public static CloudBlobDirectory cloudBlobDirectoryFrom(String connection, String containerName, String dir) throws InvalidKeyException, URISyntaxException, StorageException { CloudStorageAccount cloud = CloudStorageAccount.parse(connection); CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(containerName); container.createIfNotExists(); return container.getDirectoryReference(dir); }