Refine search
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), null); CloudBlobContainer container = blobClient.getContainerReference(containerName); for (ListBlobItem blob : container.listBlobs(prefix, true, EnumSet.of(BlobListingDetails.METADATA), null, operationContext)) { if (blob instanceof CloudBlob) { CloudBlob cloudBlob = (CloudBlob) blob; BlobProperties properties = cloudBlob.getProperties(); StorageUri uri = cloudBlob.getSnapshotQualifiedStorageUri(); .blobName(cloudBlob.getName()) .containerName(containerName) .contentType(properties.getContentType())
CloudBlobContainer container = blobClient.getContainerReference(containerName); CloudBlob blob = container.getBlockBlobReference(blobPath); blob.deleteIfExists(DeleteSnapshotsOption.valueOf(deleteSnapshotOptions), null, null, operationContext); session.transfer(flowFile, REL_SUCCESS); session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis); } catch ( StorageException | URISyntaxException e) { getLogger().error("Failed to delete the specified blob {} from Azure Storage. Routing to failure", new Object[]{blobPath}, e);
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile); CloudBlobContainer container = blobClient.getContainerReference(containerName); final CloudBlob blob = container.getBlockBlobReference(blobPath); blob.download(os, null, null, operationContext); } catch (StorageException e) { storedException.set(e); long length = blob.getProperties().getLength(); attributes.put("azure.length", String.valueOf(length)); session.getProvenanceReporter().fetch(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis); } catch (IllegalArgumentException | URISyntaxException | StorageException | ProcessException e) { if (e instanceof ProcessException && storedException.get() == null) {
public List<String> emptyCloudBlobDirectory(final String containerName, final String virtualDirPath) throws StorageException, URISyntaxException { List<String> deletedFiles = new ArrayList<>(); CloudBlobContainer container = getCloudBlobContainer(containerName); for (ListBlobItem blobItem : container.listBlobs(virtualDirPath, true, null, null, null)) { CloudBlob cloudBlob = (CloudBlob) blobItem; log.info("Removing file[%s] from Azure.", cloudBlob.getName()); if (cloudBlob.deleteIfExists()) { deletedFiles.add(cloudBlob.getName()); } } if (deletedFiles.isEmpty()) { log.warn("No files were deleted on the following Azure path: [%s]", virtualDirPath); } return deletedFiles; }
try { CloudBlobClient blobClient = AzureStorageUtils.createCloudBlobClient(context, getLogger(), flowFile); CloudBlobContainer container = blobClient.getContainerReference(containerName); CloudBlob blob = container.getBlockBlobReference(blobPath); blob.upload(in, length, null, null, operationContext); BlobProperties properties = blob.getProperties(); attributes.put("azure.container", containerName); attributes.put("azure.primaryUri", blob.getSnapshotQualifiedUri().toString()); attributes.put("azure.etag", properties.getEtag()); attributes.put("azure.length", String.valueOf(length)); session.getProvenanceReporter().send(flowFile, blob.getSnapshotQualifiedUri().toString(), transferMillis);
final BlobRequestOptions options = new BlobRequestOptions(); if(containerService.isContainer(file)) { final CloudBlobContainer container = session.getClient().getContainerReference(containerService.getContainer(file).getName()); container.setMetadata(new HashMap<String, String>(metadata)); container.uploadMetadata(AccessCondition.generateEmptyCondition(), options, context); final CloudBlob blob = session.getClient().getContainerReference(containerService.getContainer(file).getName()) .getBlobReferenceFromServer(containerService.getKey(file)); blob.downloadAttributes(); final BlobProperties properties = blob.getProperties(); if(HttpHeaders.CACHE_CONTROL.equalsIgnoreCase(m.getKey())) { blob.setMetadata(pruned); blob.uploadMetadata(AccessCondition.generateEmptyCondition(), options, context); blob.uploadProperties();
try { if(containerService.isContainer(file)) { final CloudBlobContainer container = session.getClient().getContainerReference(containerService.getContainer(file).getName()); container.downloadAttributes(); return container.getMetadata(); final CloudBlob blob = session.getClient().getContainerReference(containerService.getContainer(file).getName()) .getBlobReferenceFromServer(containerService.getKey(file)); blob.downloadAttributes(null, null, context); final Map<String, String> metadata = new HashMap<String, String>(blob.getMetadata()); final BlobProperties properties = blob.getProperties(); if(StringUtils.isNotBlank(properties.getCacheControl())) { metadata.put(HttpHeaders.CACHE_CONTROL, properties.getCacheControl());
if(containerService.isContainer(file)) { final PathAttributes attributes = new PathAttributes(); final CloudBlobContainer container = session.getClient().getContainerReference(containerService.getContainer(file).getName()); container.downloadAttributes(null, null, context); final BlobContainerProperties properties = container.getProperties(); attributes.setETag(properties.getEtag()); attributes.setModificationDate(properties.getLastModified().getTime()); final CloudBlob blob = session.getClient().getContainerReference(containerService.getContainer(file).getName()) .getBlobReferenceFromServer(containerService.getKey(file)); final BlobRequestOptions options = new BlobRequestOptions(); blob.downloadAttributes(AccessCondition.generateEmptyCondition(), options, context); final BlobProperties properties = blob.getProperties(); final PathAttributes attributes = new PathAttributes(); attributes.setSize(properties.getLength());
String lbi = "bucketinfo/" + EncyptUtils.encHashArchiveName(volumeID, Main.chunkStoreEncryptionEnabled); try { CloudBlockBlob cblob = container.getBlockBlobReference(lbi); cblob.downloadAttributes(); HashMap<String, String> md = cblob.getMetadata(); iter = container.listBlobs("claims/", true).iterator(); String suffix = "/" + vid; String prefix = "claims/"; try { CloudBlob bi = (CloudBlob) iter.next(); String nm = bi.getName(); if (nm.endsWith(suffix)) { bi.delete(); String fldr = nm.substring(0, nm.length() - suffix.length()); Iterator<ListBlobItem> _it = container.listBlobs(fldr).iterator(); if (!_it.hasNext()) { String fl = fldr.substring(prefix.length());
@Override public Path copy(final Path source, final Path copy, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { final CloudBlob target = session.getClient().getContainerReference(containerService.getContainer(copy).getName()) .getAppendBlobReference(containerService.getKey(copy)); final CloudBlob blob = session.getClient().getContainerReference(containerService.getContainer(source).getName()) .getBlobReferenceFromServer(containerService.getKey(source)); final BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(PreferencesFactory.get().getBoolean("azure.upload.md5")); final URI s = session.getHost().getCredentials().isTokenAuthentication() ? URI.create(blob.getUri().toString() + session.getHost().getCredentials().getToken()) : blob.getUri(); final String id = target.startCopy(s, AccessCondition.generateEmptyCondition(), AccessCondition.generateEmptyCondition(), options, context); if(log.isDebugEnabled()) { log.debug(String.format("Started copy for %s with copy operation ID %s", copy, id)); } // Copy original file attributes return new Path(copy.getParent(), copy.getName(), copy.getType(), new PathAttributes(source.attributes())); } catch(StorageException e) { throw new AzureExceptionMappingService().map("Cannot copy {0}", e, source); } catch(URISyntaxException e) { throw new NotfoundException(e.getMessage(), e); } }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testCloudBlobContainerReferenceFromServerSAS() throws StorageException, URISyntaxException, IOException, InvalidKeyException { this.container.create(); CloudBlob blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.BLOCK_BLOB, null, 1024, null); SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy(); Calendar now = Calendar.getInstance(); now.add(Calendar.MINUTE, 10); policy.setSharedAccessExpiryTime(now.getTime()); policy.setPermissions(EnumSet.of(SharedAccessBlobPermissions.READ)); String token = this.container.generateSharedAccessSignature(policy, null); CloudBlobContainer containerSAS = new CloudBlobContainer(this.container.getStorageUri(), new StorageCredentialsSharedAccessSignature(token)); CloudBlob blobRef = containerSAS.getBlobReferenceFromServer(blob.getName()); assertEquals(blob.getClass(), blobRef.getClass()); assertEquals(blob.getUri(), blobRef.getUri()); }
public String getNextName(String pp, long id) throws IOException { String pfx = pp + "/"; if (di == null) di = container.listBlobs(pp + "/").iterator(); while (di.hasNext()) { CloudBlob bi = (CloudBlob) di.next(); try { bi.downloadAttributes(); HashMap<String, String> md = bi.getMetadata(); boolean encrypt = Boolean.parseBoolean(md.get("encrypt")); String fname = EncyptUtils.decString(bi.getName().substring(pfx.length()), encrypt); return fname; /* * this.downloadFile(fname, new File(to.getPath() + File.separator + fname), * pp); */ } catch (Exception e) { throw new IOException(e); } } di = null; return null; }
public void deleteBlobFile(@NotNull String connectionString, @NotNull BlobFile blobFile) throws AzureCmdException { try { CloudBlobClient client = getCloudBlobClient(connectionString); String containerName = blobFile.getContainerName(); CloudBlobContainer container = client.getContainerReference(containerName); CloudBlob blob = getCloudBlob(container, blobFile); blob.deleteIfExists(); } catch (Throwable t) { throw new AzureCmdException("Error deleting the Blob File", t); } }
public void downloadBlobFileContent(@NotNull String connectionString, @NotNull BlobFile blobFile, @NotNull OutputStream content) throws AzureCmdException { try { CloudBlobClient client = getCloudBlobClient(connectionString); String containerName = blobFile.getContainerName(); CloudBlobContainer container = client.getContainerReference(containerName); CloudBlob blob = getCloudBlob(container, blobFile); blob.download(content); } catch (Throwable t) { throw new AzureCmdException("Error downloading the Blob File content", t); } }
@Override public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { final CloudBlob blob = session.getClient().getContainerReference(containerService.getContainer(file).getName()) .getBlobReferenceFromServer(containerService.getKey(file)); final BlobRequestOptions options = new BlobRequestOptions(); options.setConcurrentRequestCount(1); final BlobInputStream in = blob.openInputStream(AccessCondition.generateEmptyCondition(), options, context); if(status.isAppend()) { try {
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testCloudBlobContainerReferenceFromServerSnapshot() throws StorageException, URISyntaxException, IOException { this.container.create(); CloudBlob blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.BLOCK_BLOB, null, 1024, null); CloudBlob snapshot = blob.createSnapshot(); snapshot.getProperties().setContentType("application/octet-stream"); snapshot.getProperties().setLength(1024); CloudBlob blobRef = this.container.getBlobReferenceFromServer(snapshot.getName(), snapshot.getSnapshotID(), null, null, null); BlobTestHelper.assertAreEqual(snapshot, blobRef); }
@Override protected void readAll(final List<Address> members, final String clustername, final Responses responses) { if (clustername == null) { return; } String prefix = sanitize(clustername); Iterable<ListBlobItem> listBlobItems = containerReference.listBlobs(prefix); for (ListBlobItem blobItem : listBlobItems) { try { // If the item is a blob and not a virtual directory. // n.b. what an ugly API this is if (blobItem instanceof CloudBlob) { CloudBlob blob = (CloudBlob) blobItem; ByteArrayOutputStream os = new ByteArrayOutputStream(STREAM_BUFFER_SIZE); blob.download(os); byte[] pingBytes = os.toByteArray(); parsePingData(pingBytes, members, responses); } } catch (Exception t) { log.error("Error fetching ping data."); } } }
public static String getName(CloudBlob blob) { return Paths.get(blob.getName()).getFileName().toString(); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testCloudBlobContainerReferenceFromServer() throws StorageException, URISyntaxException, IOException { this.container.create(); CloudBlob blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.BLOCK_BLOB, null, 1024, null); blob.getProperties().setContentType("application/octet-stream"); blob.getProperties().setLength(1024); CloudBlob blobRef = this.container.getBlobReferenceFromServer(blob.getName()); BlobTestHelper.assertAreEqual(blob, blobRef); blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.PAGE_BLOB, null, 1024, null); blob.getProperties().setContentType("application/octet-stream"); blob.getProperties().setLength(1024); blobRef = this.container.getBlobReferenceFromServer(blob.getName()); BlobTestHelper.assertAreEqual(blob, blobRef); blob = BlobTestHelper.uploadNewBlob(this.container, BlobType.APPEND_BLOB, null, 1024, null); blob.getProperties().setContentType("application/octet-stream"); blob.getProperties().setLength(1024); blobRef = this.container.getBlobReferenceFromServer(blob.getName()); BlobTestHelper.assertAreEqual(blob, blobRef); }
AzureSegmentArchiveReader(CloudBlobDirectory archiveDirectory, IOMonitor ioMonitor) throws IOException { this.archiveDirectory = archiveDirectory; this.ioMonitor = ioMonitor; long length = 0; for (CloudBlob blob : AzureUtilities.getBlobs(archiveDirectory).collect(Collectors.toList())) { Map<String, String> metadata = blob.getMetadata(); if (AzureBlobMetadata.isSegment(metadata)) { AzureSegmentArchiveEntry indexEntry = AzureBlobMetadata.toIndexEntry(metadata, (int) blob.getProperties().getLength()); index.put(new UUID(indexEntry.getMsb(), indexEntry.getLsb()), indexEntry); } length += blob.getProperties().getLength(); } this.length = length; }