/** * Uses the concurrent request count from the specified client if <code>null</code>, sets a default value for * everything else, and sets defaults as defined in the parent class. * * @param options * The input options to copy from when applying defaults * @param client * A {@link CloudFileClient} object that represents the service client used to set the default timeout * interval and retry policy, if they are <code>null</code>. Additionally, the default value of * {@link #concurrentRequestCount} is <code>1</code>. * @param setStartTime * whether to initialize the startTimeInMs field, or not */ protected static final FileRequestOptions populateAndApplyDefaults(final FileRequestOptions options, final CloudFileClient client, final boolean setStartTime) { FileRequestOptions modifiedOptions = new FileRequestOptions(options); FileRequestOptions.populateRequestOptions(modifiedOptions, client.getDefaultRequestOptions(), setStartTime); FileRequestOptions.applyDefaults(modifiedOptions); return modifiedOptions; }
/** * Creates an instance of the <code>FileRequestOptions</code> class by copying values from another * <code>FileRequestOptions</code> instance. * * @param other * A {@link FileRequestOptions} object which represents the file request options to copy. */ public FileRequestOptions(final FileRequestOptions other) { super(other); if (other != null) { this.setConcurrentRequestCount(other.getConcurrentRequestCount()); this.setDisableContentMD5Validation(other.getDisableContentMD5Validation()); this.setStoreFileContentMD5(other.getStoreFileContentMD5()); this.setUseTransactionalContentMD5(other.getUseTransactionalContentMD5()); } }
@DoesServiceRequest private boolean exists(final boolean primaryOnly, final AccessCondition accessCondition, FileRequestOptions options, OperationContext opContext) throws StorageException { if (opContext == null) { opContext = new OperationContext(); } opContext.initialize(); options = FileRequestOptions.populateAndApplyDefaults(options, this.fileServiceClient); return ExecutionEngine.executeWithRetry(this.fileServiceClient, this, this.existsImpl(primaryOnly, accessCondition, options), options.getRetryPolicyFactory(), opContext); }
/** * Populates any null fields in the first requestOptions object with values from the second requestOptions object. */ private static void populateRequestOptions(FileRequestOptions modifiedOptions, final FileRequestOptions clientOptions, boolean setStartTime) { RequestOptions.populateRequestOptions(modifiedOptions, clientOptions, setStartTime); if (modifiedOptions.getConcurrentRequestCount() == null) { modifiedOptions.setConcurrentRequestCount(clientOptions.getConcurrentRequestCount()); } }
@Test @Category({ DevFabricTests.class, DevStoreTests.class }) public void testVerifyTransactionalMD5ValidationMissingOverallMD5() throws URISyntaxException, StorageException, IOException { final String fileName = FileTestHelper.generateRandomFileName(); final CloudFile fileRef = this.share.getRootDirectoryReference().getFileReference(fileName); final int length = 3*1024; ByteArrayInputStream srcStream = BlobTestHelper.getRandomDataStream(length); FileRequestOptions options = new FileRequestOptions(); options.setDisableContentMD5Validation(true); options.setStoreFileContentMD5(false); fileRef.upload(srcStream, length, null, options, null); options.setDisableContentMD5Validation(false); options.setStoreFileContentMD5(true); options.setUseTransactionalContentMD5(true); final CloudFile fileRef2 = this.share.getRootDirectoryReference().getFileReference(fileName); fileRef2.downloadRange(1024, (long)1024, new ByteArrayOutputStream(), null, options, null); assertNull(fileRef2.getProperties().getContentMD5()); }
@Test public void testFileContentMD5NewFileTest() throws URISyntaxException, StorageException, IOException { final String fileName = FileTestHelper.generateRandomFileName(); final CloudFile file = this.share.getRootDirectoryReference().getFileReference(fileName); FileRequestOptions options = new FileRequestOptions(); options.setStoreFileContentMD5(true); options.setDisableContentMD5Validation(false); File tempFile = File.createTempFile("sourceFile", ".tmp"); file.uploadFromFile(tempFile.getAbsolutePath(), null, options, null); }
FileRequestOptions options = new FileRequestOptions(); options.setDisableContentMD5Validation(true);
this.accessCondition = accessCondition; this.parentFileRef = parentFile; this.options = new FileRequestOptions(options); this.outBuffer = new ByteArrayOutputStream(); this.opContext = opContext; this.streamFaulted = false; if (this.options.getConcurrentRequestCount() < 1) { throw new IllegalArgumentException("ConcurrentRequestCount"); if (this.options.getStoreFileContentMD5()) { try { this.md5Digest = MessageDigest.getInstance("MD5"); this.threadExecutor = Executors.newFixedThreadPool(this.options.getConcurrentRequestCount()); this.completionService = new ExecutorCompletionService<Void>(this.threadExecutor); this.internalWriteThreshold = (int) Math.min(this.parentFileRef.getStreamWriteSizeInBytes(), length);
final FileRequestOptions options, final OperationContext opContext) throws StorageException { this.parentFileRef = parentFile; this.options = new FileRequestOptions(options); this.opContext = opContext; this.streamFaulted = false; this.readSize = parentFile.getStreamMinimumReadSizeInBytes(); if (options.getUseTransactionalContentMD5() && this.readSize > 4 * Constants.MB) { throw new IllegalArgumentException(SR.INVALID_RANGE_CONTENT_MD5_HEADER); this.validateFileMd5 = !options.getDisableContentMD5Validation() && !Utility.isNullOrEmpty(this.retrievedContentMD5Value);
fileRef.create(fileLengthToUse); FileRequestOptions options = new FileRequestOptions(); options.setUseTransactionalContentMD5(true);
final ByteArrayInputStream sourceStream = new ByteArrayInputStream(buff); final FileRequestOptions options = new FileRequestOptions(); final OperationContext operationContext = new OperationContext(); options.setTimeoutIntervalInMs(90000); options.setRetryPolicyFactory(new RetryNoRetry()); fileRef.upload(sourceStream, fileLength, null, options, operationContext);
@Test @Category({ DevFabricTests.class, DevStoreTests.class, SecondaryTests.class }) public void testFileMaximumExecutionTime() throws URISyntaxException, StorageException { OperationContext opContext = new OperationContext(); setDelay(opContext, 2500); opContext.getResponseReceivedEventHandler().addListener(new StorageEvent<ResponseReceivedEvent>() { @Override public void eventOccurred(ResponseReceivedEvent eventArg) { // Set status code to 500 to force a retry eventArg.getRequestResult().setStatusCode(500); } }); // set the maximum execution time FileRequestOptions options = new FileRequestOptions(); options.setMaximumExecutionTimeInMs(2000); options.setTimeoutIntervalInMs(1000); CloudFileClient fileClient = TestHelper.createCloudFileClient(); CloudFileShare share = fileClient.getShareReference(generateRandomName("share")); try { // 1. download attributes will fail as the share does not exist // 2. the executor will attempt to retry as we set the status code to 500 // 3. maximum execution time should prevent the retry from being made share.downloadAttributes(null, options, opContext); fail("Maximum execution time was reached but request did not fail."); } catch (StorageException e) { assertEquals(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION, e.getMessage()); } }
if (!options.getDisableContentMD5Validation() && options.getUseTransactionalContentMD5() && Utility.isNullOrEmpty(this.getContentMD5())) { throw new StorageException(StorageErrorCodeStrings.MISSING_MD5_HEADER, SR.MISSING_MD5,
options = FileRequestOptions.populateAndApplyDefaults(options, this.fileServiceClient, false /* setStartTime */); if (options.getStoreFileContentMD5()) { throw new IllegalArgumentException(SR.FILE_MD5_NOT_POSSIBLE);
options = FileRequestOptions.populateAndApplyDefaults(options, this.fileServiceClient); if (options.getUseTransactionalContentMD5()) { try { final MessageDigest digest = MessageDigest.getInstance("MD5");
protected static void doDownloadRangeToByteArrayTest(CloudFile file, int fileSize, int bufferSize, int bufferOffset, Long fileOffset, Long length) throws IOException, StorageException, URISyntaxException { final Random randGenerator = new Random(); final byte[] buffer = new byte[fileSize]; randGenerator.nextBytes(buffer); byte[] resultBuffer = new byte[bufferSize]; int downloadLength; FileRequestOptions options = new FileRequestOptions(); file.upload(new ByteArrayInputStream(buffer), buffer.length); downloadLength = file.downloadRangeToByteArray(fileOffset, length, resultBuffer, bufferOffset, null, options, null); int downloadSize = Math.min(fileSize - (int) (fileOffset != null ? fileOffset : 0), bufferSize - bufferOffset); if (length != null && length < downloadSize) { downloadSize = length.intValue(); } Assert.assertEquals(downloadSize, downloadLength); for (int i = 0; i < bufferOffset; i++) { Assert.assertEquals(0, resultBuffer[i]); } for (int j = 0; j < downloadLength; j++) { Assert.assertEquals(buffer[(int) ((fileOffset != null ? fileOffset : 0) + j)], resultBuffer[bufferOffset + j]); } for (int k = bufferOffset + downloadLength; k < bufferSize; k++) { Assert.assertEquals(0, resultBuffer[k]); } }
@Override public void setHeaders(HttpURLConnection connection, CloudFile file, OperationContext context) { if (operationType == FileRangeOperationType.UPDATE) { if (options.getUseTransactionalContentMD5()) { connection.setRequestProperty(Constants.HeaderConstants.CONTENT_MD5, md5); } } }
if (this.options.getStoreFileContentMD5()) { this.md5Digest.update(data, offset, nextWrite);
/** * Creates an instance of the <code>CloudFileClient</code> class using the specified File service endpoint and * account credentials. * * @param storageUri * A {@link StorageUri} object that represents the File service endpoint used to create the * client. * @param credentials * A {@link StorageCredentials} object that represents the account credentials. */ public CloudFileClient(StorageUri storageUri, StorageCredentials credentials) { super(storageUri, credentials); if (credentials == null || credentials.getClass().equals(StorageCredentialsAnonymous.class)) { throw new IllegalArgumentException(SR.STORAGE_CREDENTIALS_NULL_OR_ANONYMOUS); } FileRequestOptions.applyDefaults(this.defaultRequestOptions); }
@Override public Integer postProcessResponse(HttpURLConnection connection, CloudFile file, CloudFileClient client, OperationContext context, Integer storageObject) throws Exception { final Boolean validateMD5 = !options.getDisableContentMD5Validation() && !Utility.isNullOrEmpty(this.getContentMD5()); final String contentLength = connection.getHeaderField(Constants.HeaderConstants.CONTENT_LENGTH); final long expectedLength = Long.parseLong(contentLength); Logger.info(context, String.format(SR.CREATING_NETWORK_STREAM, expectedLength)); final NetworkInputStream streamRef = new NetworkInputStream(connection.getInputStream(), expectedLength); try { // writeToOutputStream will update the currentRequestByteCount on this request in case a retry // is needed and download should resume from that point final StreamMd5AndLength descriptor = Utility.writeToOutputStream(streamRef, outStream, -1, false, validateMD5, context, options, true, this, this.getCurrentDescriptor()); // length was already checked by the NetworkInputStream, now check Md5 if (validateMD5 && !this.getContentMD5().equals(descriptor.getMd5())) { throw new StorageException(StorageErrorCodeStrings.INVALID_MD5, String.format( SR.FILE_HASH_MISMATCH, this.getContentMD5(), descriptor.getMd5()), Constants.HeaderConstants.HTTP_UNUSED_306, null, null); } } finally { // Close the stream and return. Closing an already closed stream is harmless. So its fine to try // to drain the response and close the stream again in the executor. streamRef.close(); } return null; }