CloudBlobDirectory dir = container.GetDirectoryReference("yourFolder"); var blobs = dir.ListBlobs(new BlobRequestOptions() { UseFlatBlobListing = true });
/** * Get the list of block entries. It is used for testing purposes only. * @return List of block entries. */ @VisibleForTesting List<BlockEntry> getBlockList() throws StorageException, IOException { return blob.downloadBlockList( BlockListingFilter.COMMITTED, new BlobRequestOptions(), opContext); }
public static BlobRequestOptions withMD5Checking() { BlobRequestOptions options = new BlobRequestOptions(); options.setUseTransactionalContentMD5(true); return options; } }
private BlobRequestOptions getDownloadOptions() { BlobRequestOptions options = new BlobRequestOptions(); options.setRetryPolicyFactory( new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries)); options.setUseTransactionalContentMD5(getUseTransactionalContentMD5()); return options; }
/** * Uses the concurrent request count from the specified client if <code>null</code>, sets a default value for * everything else, and sets defaults as defined in the parent class. * * @param options * The input options to copy from when applying defaults * @param blobType * BlobType of the current operation * @param client * A {@link CloudBlobClient} object that represents the service client used to set the default timeout * interval and retry policy, if they are <code>null</code>. Additionally, the default value of * {@link #concurrentRequestCount} is <code>1</code>. * @param setStartTime * whether to initialize the startTimeInMs field, or not */ protected static final BlobRequestOptions populateAndApplyDefaults(final BlobRequestOptions options, final BlobType blobType, final CloudBlobClient client, final boolean setStartTime) { BlobRequestOptions modifiedOptions = new BlobRequestOptions(options); BlobRequestOptions.populateRequestOptions(modifiedOptions, client.getDefaultRequestOptions(), setStartTime); BlobRequestOptions.applyDefaults(modifiedOptions, blobType); return modifiedOptions; }
private BlobRequestOptions getUploadOptions() { BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(sessionConfiguration.getBoolean( KEY_STORE_BLOB_MD5, false)); options.setUseTransactionalContentMD5(getUseTransactionalContentMD5()); options.setConcurrentRequestCount(concurrentWrites); options.setRetryPolicyFactory(new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries)); return options; }
/** * Uses the concurrent request count from the specified client if <code>null</code>, sets a default value for * everything else, and sets defaults as defined in the parent class. * * @param options * The input options to copy from when applying defaults * @param blobType * BlobType of the current operation * @param client * A {@link CloudBlobClient} object that represents the service client used to set the default timeout * interval and retry policy, if they are <code>null</code>. Additionally, if the * {@link #concurrentRequestCount} field's value is null, it will be set to the value specified by the * cloud blob client's {@link CloudBlobClient#getConcurrentRequestCount} method. * @param setStartTime * whether to initialize the startTimeInMs field, or not */ protected static final BlobRequestOptions populateAndApplyDefaults(final BlobRequestOptions options, final BlobType blobType, final CloudBlobClient client, final boolean setStartTime) { BlobRequestOptions modifiedOptions = new BlobRequestOptions(options); BlobRequestOptions.populateRequestOptions(modifiedOptions, client.getDefaultRequestOptions(), setStartTime); BlobRequestOptions.applyDefaults(modifiedOptions, blobType); return modifiedOptions; }
BlobRequestOptions blobOptions = new BlobRequestOptions(); blobOptions.setRetryPolicyFactory(retryPolicy); blobClient.setDefaultRequestOptions(blobOptions);
protected static void doDownloadRangeToByteArrayTest(CloudBlob blob, int blobSize, int bufferSize, int bufferOffset, Long blobOffset, Long length) throws IOException, StorageException { final Random randGenerator = new Random(); final byte[] buffer = new byte[blobSize]; randGenerator.nextBytes(buffer); byte[] resultBuffer = new byte[bufferSize]; int downloadLength; BlobRequestOptions options = new BlobRequestOptions(); blob.upload(new ByteArrayInputStream(buffer), buffer.length); downloadLength = blob.downloadRangeToByteArray(blobOffset, length, resultBuffer, bufferOffset, null, options, null); int downloadSize = Math.min(blobSize - (int) (blobOffset != null ? blobOffset : 0), bufferSize - bufferOffset); if (length != null && length < downloadSize) { downloadSize = length.intValue(); } assertEquals(downloadSize, downloadLength); for (int i = 0; i < bufferOffset; i++) { assertEquals(0, resultBuffer[i]); } for (int j = 0; j < downloadLength; j++) { assertEquals(buffer[(int) ((blobOffset != null ? blobOffset : 0) + j)], resultBuffer[bufferOffset + j]); } for (int k = bufferOffset + downloadLength; k < bufferSize; k++) { assertEquals(0, resultBuffer[k]); } }
protected static void doDownloadTest(CloudBlob blob, int blobSize, int bufferSize, int bufferOffset) throws StorageException, IOException { final Random randGenerator = new Random(); final byte[] buffer = new byte[blobSize]; randGenerator.nextBytes(buffer); byte[] resultBuffer = new byte[bufferSize]; BlobRequestOptions options = new BlobRequestOptions(); if (blob.getClass() == CloudBlockBlob.class) { options.setUseTransactionalContentMD5(true); } blob.upload(new ByteArrayInputStream(buffer), buffer.length); blob.downloadToByteArray(resultBuffer, bufferOffset, null, options, null); for (int i = 0; i < blob.getProperties().getLength(); i++) { assertEquals(buffer[i], resultBuffer[bufferOffset + i]); } if (bufferOffset + blobSize < bufferSize) { for (int k = bufferOffset + blobSize; k < bufferSize; k++) { assertEquals(0, resultBuffer[k]); } } }
public void testWritesConcurrency() throws URISyntaxException, StorageException, IOException { int writes = 10; BlobRequestOptions options = new BlobRequestOptions(); options.setConcurrentRequestCount(5); this.smallPutThresholdHelper(Constants.MB, writes, options); this.writeFlushHelper(512, writes, options, 1); this.writeFlushHelper(512, writes, options, 4); this.writeFlushHelper(512, writes, options, writes+1); }
@Test public void testDefaultProxy() throws URISyntaxException, StorageException { CloudBlobClient blobClient = TestHelper.createCloudBlobClient(); CloudBlobContainer container = blobClient.getContainerReference("container1"); // Use a default proxy OperationContext.setDefaultProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("10.1.1.1", 8888))); // Turn off retries to make the failure happen faster BlobRequestOptions opt = new BlobRequestOptions(); opt.setRetryPolicyFactory(new RetryNoRetry()); // Unfortunately HttpURLConnection doesn't expose a getter and the usingProxy method it does have doesn't // work as one would expect and will always for us return false. So, we validate by making sure the request // fails when we set a bad proxy rather than check the proxy setting itself succeeding. try { container.exists(null, opt, null); fail("Bad proxy should throw an exception."); } catch (StorageException e) { if (e.getCause().getClass() != ConnectException.class && e.getCause().getClass() != SocketTimeoutException.class && e.getCause().getClass() != SocketException.class) { Assert.fail("Unepected exception for bad proxy"); } } }
@Test public void testProxy() throws URISyntaxException, StorageException { CloudBlobClient blobClient = TestHelper.createCloudBlobClient(); CloudBlobContainer container = blobClient.getContainerReference("container1"); // Use a request-level proxy OperationContext opContext = new OperationContext(); opContext.setProxy(new Proxy(Proxy.Type.HTTP, new InetSocketAddress("10.1.1.1", 8888))); // Turn of retries to make the failure happen faster BlobRequestOptions opt = new BlobRequestOptions(); opt.setRetryPolicyFactory(new RetryNoRetry()); // Unfortunately HttpURLConnection doesn't expose a getter and the usingProxy method it does have doesn't // work as one would expect and will always for us return false. So, we validate by making sure the request // fails when we set a bad proxy rather than check the proxy setting itself. try { container.exists(null, opt, opContext); fail("Bad proxy should throw an exception."); } catch (StorageException e) { if (e.getCause().getClass() != ConnectException.class && e.getCause().getClass() != SocketTimeoutException.class && e.getCause().getClass() != SocketException.class) { Assert.fail("Unepected exception for bad proxy"); } } }
@Override public Path copy(final Path source, final Path copy, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { try { final CloudBlob target = session.getClient().getContainerReference(containerService.getContainer(copy).getName()) .getAppendBlobReference(containerService.getKey(copy)); final CloudBlob blob = session.getClient().getContainerReference(containerService.getContainer(source).getName()) .getBlobReferenceFromServer(containerService.getKey(source)); final BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(PreferencesFactory.get().getBoolean("azure.upload.md5")); final URI s = session.getHost().getCredentials().isTokenAuthentication() ? URI.create(blob.getUri().toString() + session.getHost().getCredentials().getToken()) : blob.getUri(); final String id = target.startCopy(s, AccessCondition.generateEmptyCondition(), AccessCondition.generateEmptyCondition(), options, context); if(log.isDebugEnabled()) { log.debug(String.format("Started copy for %s with copy operation ID %s", copy, id)); } // Copy original file attributes return new Path(copy.getParent(), copy.getName(), copy.getType(), new PathAttributes(source.attributes())); } catch(StorageException e) { throw new AzureExceptionMappingService().map("Cannot copy {0}", e, source); } catch(URISyntaxException e) { throw new NotfoundException(e.getMessage(), e); } }
@Override public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException { ResultSegment<CloudBlobContainer> result; ResultContinuation token = null; try { final AttributedList<Path> containers = new AttributedList<Path>(); do { final BlobRequestOptions options = new BlobRequestOptions(); result = session.getClient().listContainersSegmented(null, ContainerListingDetails.NONE, preferences.getInteger("azure.listing.chunksize"), token, options, context); for(CloudBlobContainer container : result.getResults()) { final PathAttributes attributes = new PathAttributes(); attributes.setETag(container.getProperties().getEtag()); attributes.setModificationDate(container.getProperties().getLastModified().getTime()); containers.add(new Path(PathNormalizer.normalize(container.getName()), EnumSet.of(Path.Type.volume, Path.Type.directory), attributes)); } listener.chunk(directory, containers); token = result.getContinuationToken(); } while(result.getHasMoreResults()); return containers; } catch(StorageException e) { throw new AzureExceptionMappingService().map("Listing directory {0} failed", e, directory); } }
/** * Test requesting stored content MD5 with OpenWriteExisting(). * * @throws URISyntaxException * @throws StorageException */ @Test public void testPageOpenWriteExistingWithMD5() throws URISyntaxException, StorageException, IOException { final String pageBlobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testPageBlob"); final CloudPageBlob pageBlobRef = this.container.getPageBlobReference(pageBlobName); pageBlobRef.create(512); BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(true); options.setDisableContentMD5Validation(false); try { pageBlobRef.openWriteExisting(null, options, null); fail("Expect failure due to requesting MD5 calculation"); } catch (IllegalArgumentException e) { } }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testBlobUploadFromStreamRequestOptionsTest() throws URISyntaxException, StorageException, IOException { final String blockBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef1 = this.container.getBlockBlobReference(blockBlobName1); final String blockBlobName2 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef2 = this.container.getBlockBlobReference(blockBlobName2); final int length = 2 * com.microsoft.azure.storage.Constants.MB; ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length); BlobRequestOptions options = new BlobRequestOptions(); options.setSingleBlobPutThresholdInBytes(length / 2); options.setRetryPolicyFactory(RetryNoRetry.getInstance()); OperationContext context = new OperationContext(); blockBlobRef1.upload(srcStream, length, null /* accessCondition */, options, context); assertTrue(context.getRequestResults().size() >= 2); srcStream.reset(); options.setSingleBlobPutThresholdInBytes(length); context = new OperationContext(); blockBlobRef2.upload(srcStream, length, null /* accessCondition */, options, context); assertTrue(context.getRequestResults().size() <= 2); }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testVerifyTransactionalMD5ValidationMissingOverallMD5() throws URISyntaxException, StorageException, IOException { final String blockBlobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef = this.container.getBlockBlobReference(blockBlobName); final int length = 2 * 1024 * 1024; ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length); BlobRequestOptions options = new BlobRequestOptions(); options.setSingleBlobPutThresholdInBytes(1024*1024); options.setDisableContentMD5Validation(true); options.setStoreBlobContentMD5(false); blockBlobRef.upload(srcStream, -1, null, options, null); options.setDisableContentMD5Validation(false); options.setStoreBlobContentMD5(true); options.setUseTransactionalContentMD5(true); final CloudBlockBlob blockBlobRef2 = this.container.getBlockBlobReference(blockBlobName); blockBlobRef2.downloadRange(1024, (long)1024, new ByteArrayOutputStream(), null, options, null); assertNull(blockBlobRef2.getProperties().getContentMD5()); }
private static void testContainerDownloadAttributes(LocationMode optionsLocationMode, LocationMode clientLocationMode, StorageLocation initialLocation, List<RetryContext> retryContextList, List<RetryInfo> retryInfoList) throws URISyntaxException, StorageException { CloudBlobContainer container = BlobTestHelper.getRandomContainerReference(); MultiLocationTestHelper helper = new MultiLocationTestHelper(container.getServiceClient().getStorageUri(), initialLocation, retryContextList, retryInfoList); container.getServiceClient().getDefaultRequestOptions().setLocationMode(clientLocationMode); BlobRequestOptions options = new BlobRequestOptions(); options.setLocationMode(optionsLocationMode); options.setRetryPolicyFactory(helper.retryPolicy); try { container.downloadAttributes(null, options, helper.operationContext); } catch (StorageException ex) { assertEquals(HttpURLConnection.HTTP_NOT_FOUND, ex.getHttpStatusCode()); } finally { helper.close(); } }
@Test public void testUploadDownloadBlobProperties() throws URISyntaxException, StorageException, IOException { final int length = 512; // do this to make sure the set MD5 can be compared without an exception being thrown BlobRequestOptions options = new BlobRequestOptions(); options.setDisableContentMD5Validation(true); // with explicit upload/download of properties String pageBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); CloudPageBlob pageBlobRef1 = this.container.getPageBlobReference(pageBlobName1); pageBlobRef1.upload(BlobTestHelper.getRandomDataStream(length), length); BlobTestHelper.setBlobProperties(pageBlobRef1); BlobProperties props1 = pageBlobRef1.getProperties(); pageBlobRef1.uploadProperties(); pageBlobRef1.downloadAttributes(null, options, null); BlobProperties props2 = pageBlobRef1.getProperties(); BlobTestHelper.assertAreEqual(props1, props2); // by uploading/downloading the blob pageBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); pageBlobRef1 = this.container.getPageBlobReference(pageBlobName1); BlobTestHelper.setBlobProperties(pageBlobRef1); props1 = pageBlobRef1.getProperties(); pageBlobRef1.upload(BlobTestHelper.getRandomDataStream(length), length); pageBlobRef1.download(new ByteArrayOutputStream(), null, options, null); props2 = pageBlobRef1.getProperties(); BlobTestHelper.assertAreEqual(props1, props2); }