/** * Sets the object metadata to use for the new, copied object and returns * this object, enabling additional method calls to be chained together. By * default, the object metadata from the source object will be copied to the * destination object, but if callers set object metadata with this method, * it will be used instead. * * @param newObjectMetadata * The object metadata to use for the newly copied object. * * @return This <code>CopyObjectRequest</code>, enabling additional method calls to be * chained together. */ public CopyObjectRequest withNewObjectMetadata(ObjectMetadata newObjectMetadata) { setNewObjectMetadata(newObjectMetadata); return this; }
@Override protected boolean copyObject(String src, String dst) { LOG.debug("Copying {} to {}", src, dst); // Retry copy for a few times, in case some AWS internal errors happened during copy. int retries = 3; for (int i = 0; i < retries; i++) { try { CopyObjectRequest request = new CopyObjectRequest(mBucketName, src, mBucketName, dst); if (Boolean.parseBoolean( mConf.get(PropertyKey.UNDERFS_S3A_SERVER_SIDE_ENCRYPTION_ENABLED))) { ObjectMetadata meta = new ObjectMetadata(); meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setNewObjectMetadata(meta); } mManager.copy(request).waitForCopyResult(); return true; } catch (AmazonClientException | InterruptedException e) { LOG.error("Failed to copy file {} to {}", src, dst, e); if (i != retries - 1) { LOG.error("Retrying copying file {} to {}", src, dst); } } } LOG.error("Failed to copy file {} to {}, after {} retries", src, dst, retries); return false; }
/** * Sets the object metadata to use for the new, copied object and returns * this object, enabling additional method calls to be chained together. By * default, the object metadata from the source object will be copied to the * destination object, but if callers set object metadata with this method, * it will be used instead. * * @param newObjectMetadata The object metadata to use for the newly copied * object. * @return This <code>CopyObjectRequest</code>, enabling additional method * calls to be chained together. */ public CopyObjectRequest withNewObjectMetadata(ObjectMetadata newObjectMetadata) { setNewObjectMetadata(newObjectMetadata); return this; }
/** * Sets the object metadata to use for the new, copied object and returns * this object, enabling additional method calls to be chained together. By * default, the object metadata from the source object will be copied to the * destination object, but if callers set object metadata with this method, * it will be used instead. * * @param newObjectMetadata * The object metadata to use for the newly copied object. * * @return This <code>CopyObjectRequest</code>, enabling additional method calls to be * chained together. */ public CopyObjectRequest withNewObjectMetadata(ObjectMetadata newObjectMetadata) { setNewObjectMetadata(newObjectMetadata); return this; }
/** * Sets the object metadata to use for the new, copied object and returns * this object, enabling additional method calls to be chained together. By * default, the object metadata from the source object will be copied to the * destination object, but if callers set object metadata with this method, * it will be used instead. * * @param newObjectMetadata The object metadata to use for the newly copied * object. * @return This <code>CopyObjectRequest</code>, enabling additional method * calls to be chained together. */ public CopyObjectRequest withNewObjectMetadata(ObjectMetadata newObjectMetadata) { setNewObjectMetadata(newObjectMetadata); return this; }
private void applyObjectMetadata(CopyObjectRequest copyObjectRequest) { if (s3s3CopierOptions.isS3ServerSideEncryption()) { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); copyObjectRequest.setNewObjectMetadata(objectMetadata); } }
private void applyObjectMetadata(CopyObjectRequest copyObjectRequest) { if (s3s3CopierOptions.isS3ServerSideEncryption()) { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); copyObjectRequest.setNewObjectMetadata(objectMetadata); } }
@Override public void touch(DataIdentifier identifier, long minModifiedDate) throws DataStoreException { ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { final long start = System.currentTimeMillis(); final String key = getKeyName(identifier); if (minModifiedDate > 0 && minModifiedDate > getLastModified(identifier)) { CopyObjectRequest copReq = new CopyObjectRequest(bucket, key, bucket, key); copReq.setNewObjectMetadata(new ObjectMetadata()); Copy copy = tmx.copy(s3ReqDecorator.decorate(copReq)); copy.waitForCompletion(); LOG.debug("[{}] touched. time taken [{}] ms ", new Object[] { identifier, (System.currentTimeMillis() - start) }); } else { LOG.trace("[{}] touch not required. time taken [{}] ms ", new Object[] { identifier, (System.currentTimeMillis() - start) }); } } catch (Exception e) { throw new DataStoreException("Error occured in touching key [" + identifier.toString() + "]", e); } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
private void setS3ObjectUserProperty(BinaryKey binaryKey, String metadataKey, String metadataValue) throws BinaryStoreException { try { String key = binaryKey.toString(); ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, key); Map<String, String> userMetadata = metadata.getUserMetadata(); if(null != metadataValue && metadataValue.equals(userMetadata.get(metadataKey))) { return; // The key/value pair already exists in user metadata, skip update } userMetadata.put(metadataKey, metadataValue); metadata.setUserMetadata(userMetadata); // Update the object in place CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, key, bucketName, key); copyRequest.setNewObjectMetadata(metadata); s3Client.copyObject(copyRequest); } catch (AmazonClientException e) { throw new BinaryStoreException(e); } }
private void setS3ObjectUserProperty(BinaryKey binaryKey, String metadataKey, String metadataValue) throws BinaryStoreException { try { String key = binaryKey.toString(); ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, key); Map<String, String> userMetadata = metadata.getUserMetadata(); if(null != metadataValue && metadataValue.equals(userMetadata.get(metadataKey))) { return; // The key/value pair already exists in user metadata, skip update } userMetadata.put(metadataKey, metadataValue); metadata.setUserMetadata(userMetadata); // Update the object in place CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, key, bucketName, key); copyRequest.setNewObjectMetadata(metadata); s3Client.copyObject(copyRequest); } catch (AmazonClientException e) { throw new BinaryStoreException(e); } }
/** * Set encryption in {@link CopyObjectRequest} */ public CopyObjectRequest decorate(CopyObjectRequest request) { switch (getDataEncryption()) { case SSE_S3: ObjectMetadata metadata = request.getNewObjectMetadata() == null ? new ObjectMetadata() : request.getNewObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setNewObjectMetadata(metadata); break; case NONE: break; } return request; }
/** * Set encryption in {@link CopyObjectRequest} */ public CopyObjectRequest decorate(CopyObjectRequest request) { switch (getDataEncryption()) { case SSE_S3: ObjectMetadata metadata = request.getNewObjectMetadata() == null ? new ObjectMetadata() : request.getNewObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setNewObjectMetadata(metadata); break; case NONE: break; } return request; }
/** * Set encryption in {@link CopyObjectRequest} */ public CopyObjectRequest decorate(CopyObjectRequest request) { switch (getDataEncryption()) { case SSE_S3: ObjectMetadata metadata = request.getNewObjectMetadata() == null ? new ObjectMetadata() : request.getNewObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setNewObjectMetadata(metadata); break; case NONE: break; } return request; }
/** * Set encryption in {@link CopyObjectRequest} */ public CopyObjectRequest decorate(CopyObjectRequest request) { switch (getDataEncryption()) { case SSE_S3: ObjectMetadata metadata = request.getNewObjectMetadata() == null ? new ObjectMetadata() : request.getNewObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); request.setNewObjectMetadata(metadata); break; case NONE: break; } return request; }
@Override protected void storeMimeType(BinaryValue binaryValue, String mimeType) throws BinaryStoreException { try { String key = binaryValue.getKey().toString(); ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, key); metadata.setContentType(mimeType); metadata.addUserMetadata(USER_MIME_TYPE_KEY, String.valueOf(true)); // Update the object in place CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, key, bucketName, key); copyRequest.setNewObjectMetadata(metadata); s3Client.copyObject(copyRequest); } catch (AmazonClientException e) { throw new BinaryStoreException(e); } }
@Override protected void storeMimeType(BinaryValue binaryValue, String mimeType) throws BinaryStoreException { try { String key = binaryValue.getKey().toString(); ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, key); metadata.setContentType(mimeType); // Update the object in place CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, key, bucketName, key); copyRequest.setNewObjectMetadata(metadata); s3Client.copyObject(copyRequest); } catch (AmazonClientException e) { throw new BinaryStoreException(e); } }
@VisibleForTesting CopyObjectResult atomicCopy( S3ResourceId sourcePath, S3ResourceId destinationPath, ObjectMetadata sourceObjectMetadata) throws AmazonClientException { CopyObjectRequest copyObjectRequest = new CopyObjectRequest( sourcePath.getBucket(), sourcePath.getKey(), destinationPath.getBucket(), destinationPath.getKey()); copyObjectRequest.setNewObjectMetadata(sourceObjectMetadata); copyObjectRequest.setStorageClass(options.getS3StorageClass()); copyObjectRequest.setSourceSSECustomerKey(options.getSSECustomerKey()); copyObjectRequest.setDestinationSSECustomerKey(options.getSSECustomerKey()); return amazonS3.copyObject(copyObjectRequest); }
private void updateObjectProperties(String bucketName, String contentId, ObjectMetadata objMetadata) { try { AccessControlList originalACL = s3Client.getObjectAcl(bucketName, contentId); CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, contentId, bucketName, contentId); copyRequest.setStorageClass(DEFAULT_STORAGE_CLASS); copyRequest.setNewObjectMetadata(objMetadata); s3Client.copyObject(copyRequest); s3Client.setObjectAcl(bucketName, contentId, originalACL); } catch (AmazonClientException e) { throwIfContentNotExist(bucketName, contentId); String err = "Could not update metadata for content " + contentId + " in S3 bucket " + bucketName + " due to error: " + e.getMessage(); throw new StorageException(err, e, NO_RETRY); } }
@Override public Transfer performTransfer(TransferManager transferManager) { // Create a copy request. CopyObjectRequest copyObjectRequest = new CopyObjectRequest(params.getSourceBucketName(), params.getSourceObjectKey(), params.getTargetBucketName(), params.getTargetObjectKey()); // If KMS Key ID is specified, set the AWS Key Management System parameters to be used to encrypt the object. if (StringUtils.isNotBlank(params.getKmsKeyId())) { copyObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(params.getKmsKeyId())); } // Otherwise, specify the server-side encryption algorithm for encrypting the object using AWS-managed keys. else { ObjectMetadata metadata = new ObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); copyObjectRequest.setNewObjectMetadata(metadata); } return s3Operations.copyFile(copyObjectRequest, transferManager); } });
@Override public Transfer performTransfer(TransferManager transferManager) { // Create a copy request. CopyObjectRequest copyObjectRequest = new CopyObjectRequest(params.getSourceBucketName(), params.getSourceObjectKey(), params.getTargetBucketName(), params.getTargetObjectKey()); // If KMS Key ID is specified, set the AWS Key Management System parameters to be used to encrypt the object. if (StringUtils.isNotBlank(params.getKmsKeyId())) { copyObjectRequest.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(params.getKmsKeyId())); } // Otherwise, specify the server-side encryption algorithm for encrypting the object using AWS-managed keys. else { ObjectMetadata metadata = new ObjectMetadata(); metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); copyObjectRequest.setNewObjectMetadata(metadata); } return s3Operations.copyFile(copyObjectRequest, transferManager); } });