private BlobRequestOptions getUploadOptions() { BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(sessionConfiguration.getBoolean( KEY_STORE_BLOB_MD5, false)); options.setUseTransactionalContentMD5(getUseTransactionalContentMD5()); options.setConcurrentRequestCount(concurrentWrites); options.setRetryPolicyFactory(new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries)); return options; }
public CloudBlobContainer makeObject() throws URISyntaxException, StorageException { SDFSLogger.getLog().info("pool size is " + this.passiveObjects.size()); CloudBlobClient serviceClient = account.createCloudBlobClient(); serviceClient = account.createCloudBlobClient(); serviceClient.getDefaultRequestOptions().setConcurrentRequestCount( Main.dseIOThreads * 2); serviceClient.getDefaultRequestOptions().setTimeoutIntervalInMs( 10 * 1000); serviceClient.getDefaultRequestOptions().setRetryPolicyFactory( new RetryExponentialRetry(500, 5)); CloudBlobContainer container = serviceClient .getContainerReference(this.bucket); container.createIfNotExists(); return container; }
/** * Creates an instance of the <code>BlobRequestOptions</code> class by copying values from another * <code>BlobRequestOptions</code> instance. * * @param other * A {@link BlobRequestOptions} object which represents the blob request options to copy. */ public BlobRequestOptions(final BlobRequestOptions other) { super(other); if (other != null) { this.setAbsorbConditionalErrorsOnRetry(other.getAbsorbConditionalErrorsOnRetry()); this.setConcurrentRequestCount(other.getConcurrentRequestCount()); this.setUseTransactionalContentMD5(other.getUseTransactionalContentMD5()); this.setStoreBlobContentMD5(other.getStoreBlobContentMD5()); this.setDisableContentMD5Validation(other.getDisableContentMD5Validation()); this.setSingleBlobPutThresholdInBytes(other.getSingleBlobPutThresholdInBytes()); } }
public static BlobRequestOptions withMD5Checking() { BlobRequestOptions options = new BlobRequestOptions(); options.setUseTransactionalContentMD5(true); return options; } }
BlobRequestOptions blobOptions = new BlobRequestOptions(); blobOptions.setRetryPolicyFactory(retryPolicy); blobClient.setDefaultRequestOptions(blobOptions);
CloudBlobDirectory dir = container.GetDirectoryReference("yourFolder"); var blobs = dir.ListBlobs(new BlobRequestOptions() { UseFlatBlobListing = true });
/** * Test requesting stored content MD5 with OpenWriteExisting(). * * @throws URISyntaxException * @throws StorageException */ @Test public void testPageOpenWriteExistingWithMD5() throws URISyntaxException, StorageException, IOException { final String pageBlobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testPageBlob"); final CloudPageBlob pageBlobRef = this.container.getPageBlobReference(pageBlobName); pageBlobRef.create(512); BlobRequestOptions options = new BlobRequestOptions(); options.setStoreBlobContentMD5(true); options.setDisableContentMD5Validation(false); try { pageBlobRef.openWriteExisting(null, options, null); fail("Expect failure due to requesting MD5 calculation"); } catch (IllegalArgumentException e) { } }
CloudBlockBlob blob = getAzureContainer().getBlockBlobReference(key); if (!blob.exists()) { BlobRequestOptions options = new BlobRequestOptions(); options.setConcurrentRequestCount(concurrentRequestCount); boolean useBufferedStream = len < BUFFERED_STREAM_THRESHHOLD; final InputStream in = useBufferedStream ? new BufferedInputStream(new FileInputStream(file)) : new FileInputStream(file);
private static void testContainerDownloadAttributes(LocationMode optionsLocationMode, LocationMode clientLocationMode, StorageLocation initialLocation, List<RetryContext> retryContextList, List<RetryInfo> retryInfoList) throws URISyntaxException, StorageException { CloudBlobContainer container = BlobTestHelper.getRandomContainerReference(); MultiLocationTestHelper helper = new MultiLocationTestHelper(container.getServiceClient().getStorageUri(), initialLocation, retryContextList, retryInfoList); container.getServiceClient().getDefaultRequestOptions().setLocationMode(clientLocationMode); BlobRequestOptions options = new BlobRequestOptions(); options.setLocationMode(optionsLocationMode); options.setRetryPolicyFactory(helper.retryPolicy); try { container.downloadAttributes(null, options, helper.operationContext); } catch (StorageException ex) { assertEquals(HttpURLConnection.HTTP_NOT_FOUND, ex.getHttpStatusCode()); } finally { helper.close(); } }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testVerifyTransactionalMD5ValidationMissingOverallMD5() throws URISyntaxException, StorageException, IOException { final String blockBlobName = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef = this.container.getBlockBlobReference(blockBlobName); final int length = 2 * 1024 * 1024; ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length); BlobRequestOptions options = new BlobRequestOptions(); options.setSingleBlobPutThresholdInBytes(1024*1024); options.setDisableContentMD5Validation(true); options.setStoreBlobContentMD5(false); blockBlobRef.upload(srcStream, -1, null, options, null); options.setDisableContentMD5Validation(false); options.setStoreBlobContentMD5(true); options.setUseTransactionalContentMD5(true); final CloudBlockBlob blockBlobRef2 = this.container.getBlockBlobReference(blockBlobName); blockBlobRef2.downloadRange(1024, (long)1024, new ByteArrayOutputStream(), null, options, null); assertNull(blockBlobRef2.getProperties().getContentMD5()); }
@Test public void testAppendBlobInputStream() throws URISyntaxException, StorageException, IOException { final int blobLength = 16 * 1024; final Random randGenerator = new Random(); String blobName = BlobTestHelper .generateRandomBlobNameWithPrefix("testblob"); final CloudAppendBlob blobRef = this.container .getAppendBlobReference(blobName); final byte[] buff = new byte[blobLength]; randGenerator.nextBytes(buff); buff[0] = -1; buff[1] = -128; final ByteArrayInputStream sourceStream = new ByteArrayInputStream(buff); final BlobRequestOptions options = new BlobRequestOptions(); final OperationContext operationContext = new OperationContext(); options.setTimeoutIntervalInMs(90000); options.setRetryPolicyFactory(new RetryNoRetry()); blobRef.upload(sourceStream, blobLength, null, options, operationContext); BlobInputStream blobStream = blobRef.openInputStream(); for (int i = 0; i < blobLength; i++) { int data = blobStream.read(); assertTrue(data >= 0); assertEquals(buff[i], (byte) data); } assertEquals(-1, blobStream.read()); blobRef.delete(); }
final ByteArrayInputStream sourceStream = new ByteArrayInputStream(buff); final BlobRequestOptions options = new BlobRequestOptions(); final OperationContext operationContext = new OperationContext(); options.setStoreBlobContentMD5(true); options.setTimeoutIntervalInMs(90000); options.setRetryPolicyFactory(new RetryNoRetry()); blobRef.uploadFullBlob(sourceStream, blobLength, null, options, operationContext);
private BlobRequestOptions getDownloadOptions() { BlobRequestOptions options = new BlobRequestOptions(); options.setRetryPolicyFactory( new RetryExponentialRetry(minBackoff, deltaBackoff, maxBackoff, maxRetries)); options.setUseTransactionalContentMD5(getUseTransactionalContentMD5()); return options; }
@Test public void testUploadDownloadBlobProperties() throws URISyntaxException, StorageException, IOException { final int length = 512; // do this to make sure the set MD5 can be compared without an exception being thrown BlobRequestOptions options = new BlobRequestOptions(); options.setDisableContentMD5Validation(true); // with explicit upload/download of properties String pageBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); CloudPageBlob pageBlobRef1 = this.container.getPageBlobReference(pageBlobName1); pageBlobRef1.upload(BlobTestHelper.getRandomDataStream(length), length); BlobTestHelper.setBlobProperties(pageBlobRef1); BlobProperties props1 = pageBlobRef1.getProperties(); pageBlobRef1.uploadProperties(); pageBlobRef1.downloadAttributes(null, options, null); BlobProperties props2 = pageBlobRef1.getProperties(); BlobTestHelper.assertAreEqual(props1, props2); // by uploading/downloading the blob pageBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); pageBlobRef1 = this.container.getPageBlobReference(pageBlobName1); BlobTestHelper.setBlobProperties(pageBlobRef1); props1 = pageBlobRef1.getProperties(); pageBlobRef1.upload(BlobTestHelper.getRandomDataStream(length), length); pageBlobRef1.download(new ByteArrayOutputStream(), null, options, null); props2 = pageBlobRef1.getProperties(); BlobTestHelper.assertAreEqual(props1, props2); }
final BlobRequestOptions options = new BlobRequestOptions(); options.setConcurrentRequestCount(1); options.setStoreBlobContentMD5(preferences.getBoolean("azure.upload.md5")); final BlobOutputStream out; if(status.isAppend()) { options.setStoreBlobContentMD5(false); out = blob.openWriteExisting(AccessCondition.generateEmptyCondition(), options, context);
@Test public void testAppendBlobDownloadRangeTest() throws URISyntaxException, StorageException, IOException { byte[] buffer = BlobTestHelper.getRandomBuffer(2 * 1024); CloudAppendBlob blob = this.container.getAppendBlobReference("blob1"); ByteArrayInputStream wholeBlob = new ByteArrayInputStream(buffer); BlobRequestOptions opt = new BlobRequestOptions(); opt.setStoreBlobContentMD5(false); blob.upload(wholeBlob, 2 * 1024, null, opt, null); ByteArrayOutputStream blobStream = new ByteArrayOutputStream(); try { blob.downloadRange(0, new Long(0), blobStream); } catch (IndexOutOfBoundsException ex) { } blob.downloadRange(0, new Long(1024), blobStream); assertEquals(blobStream.size(), 1024); BlobTestHelper.assertStreamsAreEqualAtIndex(new ByteArrayInputStream( blobStream.toByteArray()), wholeBlob, 0, 0, 1024, 2 * 1024); CloudAppendBlob blob2 = this.container.getAppendBlobReference("blob1"); try { blob.downloadRange(1024, new Long(0), blobStream); } catch (IndexOutOfBoundsException ex) { } ByteArrayOutputStream blobStream2 = new ByteArrayOutputStream(); blob2.downloadRange(1024, new Long(1024), blobStream2); BlobTestHelper.assertStreamsAreEqualAtIndex(new ByteArrayInputStream( blobStream2.toByteArray()), wholeBlob, 1024, 1024, 1024, 2 * 1024); BlobTestHelper.assertAreEqual(blob, blob2); }
/** * Uses the concurrent request count from the specified client if <code>null</code>, sets a default value for * everything else, and sets defaults as defined in the parent class. * * @param options * The input options to copy from when applying defaults * @param blobType * BlobType of the current operation * @param client * A {@link CloudBlobClient} object that represents the service client used to set the default timeout * interval and retry policy, if they are <code>null</code>. Additionally, the default value of * {@link #concurrentRequestCount} is <code>1</code>. * @param setStartTime * whether to initialize the startTimeInMs field, or not */ protected static final BlobRequestOptions populateAndApplyDefaults(final BlobRequestOptions options, final BlobType blobType, final CloudBlobClient client, final boolean setStartTime) { BlobRequestOptions modifiedOptions = new BlobRequestOptions(options); BlobRequestOptions.populateRequestOptions(modifiedOptions, client.getDefaultRequestOptions(), setStartTime); BlobRequestOptions.applyDefaults(modifiedOptions, blobType); return modifiedOptions; }
this.parentBlobRef = parentBlob; this.parentBlobRef.assertCorrectBlobType(); this.options = new BlobRequestOptions(options); this.outBuffer = new ByteArrayOutputStream(); this.opContext = opContext; if (this.options.getConcurrentRequestCount() < 1) { throw new IllegalArgumentException("ConcurrentRequestCount"); this.options.getConcurrentRequestCount() == null ? 1 : this.options.getConcurrentRequestCount() * 2)); if (this.options.getStoreBlobContentMD5()) { try { this.md5Digest = MessageDigest.getInstance("MD5"); this.options.getConcurrentRequestCount(), this.options.getConcurrentRequestCount(), 10, TimeUnit.SECONDS,
this.parentBlobRef = parentBlob; this.parentBlobRef.assertCorrectBlobType(); this.options = new BlobRequestOptions(options); this.opContext = opContext; this.streamFaulted = false; this.readSize = parentBlob.getStreamMinimumReadSizeInBytes(); if (options.getUseTransactionalContentMD5() && this.readSize > 4 * Constants.MB) { throw new IllegalArgumentException(SR.INVALID_RANGE_CONTENT_MD5_HEADER); this.validateBlobMd5 = !options.getDisableContentMD5Validation() && !Utility.isNullOrEmpty(this.retrievedContentMD5Value);
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testBlobUploadFromStreamRequestOptionsTest() throws URISyntaxException, StorageException, IOException { final String blockBlobName1 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef1 = this.container.getBlockBlobReference(blockBlobName1); final String blockBlobName2 = BlobTestHelper.generateRandomBlobNameWithPrefix("testBlockBlob"); final CloudBlockBlob blockBlobRef2 = this.container.getBlockBlobReference(blockBlobName2); final int length = 2 * com.microsoft.azure.storage.Constants.MB; ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length); BlobRequestOptions options = new BlobRequestOptions(); options.setSingleBlobPutThresholdInBytes(length / 2); options.setRetryPolicyFactory(RetryNoRetry.getInstance()); OperationContext context = new OperationContext(); blockBlobRef1.upload(srcStream, length, null /* accessCondition */, options, context); assertTrue(context.getRequestResults().size() >= 2); srcStream.reset(); options.setSingleBlobPutThresholdInBytes(length); context = new OperationContext(); blockBlobRef2.upload(srcStream, length, null /* accessCondition */, options, context); assertTrue(context.getRequestResults().size() <= 2); }