@Override public void deleteRemoteFiles(List<Path> remotePaths) throws BackupRestoreException { if (remotePaths.isEmpty()) return; try { List<DeleteObjectsRequest.KeyVersion> keys = remotePaths .stream() .map( remotePath -> new DeleteObjectsRequest.KeyVersion( remotePath.toString())) .collect(Collectors.toList()); s3Client.deleteObjects( new DeleteObjectsRequest(getShard()).withKeys(keys).withQuiet(true)); logger.info("Deleted {} objects from S3", remotePaths.size()); } catch (Exception e) { logger.error("Error while trying to delete the objects from S3: {}", e.getMessage()); throw new BackupRestoreException(e + " while trying to delete the objects"); } }
try { DeleteObjectsRequest dor = new DeleteObjectsRequest(bucket_name) .withKeys(object_keys); s3.deleteObjects(dor); } catch (AmazonServiceException e) {
@Override protected List<String> deleteObjects(List<String> keys) throws IOException { if (!mAlluxioConf.getBoolean(PropertyKey.UNDERFS_S3A_BULK_DELETE_ENABLED)) { return super.deleteObjects(keys); } Preconditions.checkArgument(keys != null && keys.size() <= getListingChunkLengthMax()); try { List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>(); for (String key : keys) { keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); } DeleteObjectsResult deletedObjectsResult = mClient.deleteObjects(new DeleteObjectsRequest(mBucketName).withKeys(keysToDelete)); List<String> deletedObjects = new ArrayList<>(); for (DeleteObjectsResult.DeletedObject deletedObject : deletedObjectsResult .getDeletedObjects()) { deletedObjects.add(deletedObject.getKey()); } return deletedObjects; } catch (AmazonClientException e) { throw new IOException(e); } }
private static void deleteByFullPathPrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); String[] keys = listByFullPathPrefix(bucket, s3prefix).toArray(new String[] {}); if (keys.length > 0) { DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(keys); AmazonS3ClientBuilder.defaultClient().deleteObjects(request); } }
private void delete(String bucket, Collection<String> keys) throws IOException { checkArgument( keys.size() <= MAX_DELETE_OBJECTS_PER_REQUEST, "only %s keys can be deleted per request, but got %s", MAX_DELETE_OBJECTS_PER_REQUEST, keys.size()); List<KeyVersion> deleteKeyVersions = keys.stream().map(KeyVersion::new).collect(Collectors.toList()); DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(deleteKeyVersions); try { amazonS3.deleteObjects(request); } catch (AmazonClientException e) { throw new IOException(e); } }
/** * Delete files. * * @param s3client the s 3 client * @param s3Bucket the s 3 bucket * @param folder the folder */ private void deleteFiles(AmazonS3 s3client,String s3Bucket,String folder){ String[] keys = listKeys(s3client,s3Bucket,folder); DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(s3Bucket).withKeys((keys)); try{ DeleteObjectsResult result = s3client.deleteObjects(multiObjectDeleteRequest); log.debug("Files Deleted " +result.getDeletedObjects().stream().map(obj->obj.getKey()).collect(Collectors.toList())); }catch(Exception e){ log.error("Delete Failed",e); } }
/** * Deletes the keys for the given S3 URI. The URI can represent a prefix or a single key. This method will block * on an existing async sync to S3 before running. * @param s3Uri - The S3 URI (key or prefix) to delete */ public void deleteKeys(URI s3Uri) throws InterruptedException, ExecutionException { waitForSyncToS3(); Preconditions.checkNotNull(s3Uri); Preconditions.checkArgument(SCHEME.equalsIgnoreCase(s3Uri.getScheme())); List<S3ObjectSummary> summaries = listKeys(s3Uri); if(summaries.size() > 0) { String bucket = s3Uri.getHost(); List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(); for (S3ObjectSummary summary : summaries) { keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); if(keys.size() >= MAX_KEYS_PER_S3_OP) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteObjectsRequest); keys = new ArrayList<>(); } } if(keys.size() > 0) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteObjectsRequest); } } }
@Override public Void call() { s3.deleteObjects(new DeleteObjectsRequest(config.getBucketName()).withKeys(deleteVersions)); return null; } }, OPERATION_DELETE_OBJECTS);
@Override public void deleteObjects(final Collection<String> keys) { if (keys.isEmpty()) { logger.info("AWS S3 bucket [{}] is empty. Nothing to delete.", awsS3Bucket.getName()); return; } final List<DeleteObjectsRequest.KeyVersion> keyVersions = keys.stream() .map(DeleteObjectsRequest.KeyVersion::new) .collect(Collectors.toList()); try { DeleteObjectsRequest dor = new DeleteObjectsRequest(awsS3Bucket.getName()) .withKeys(keyVersions); amazonS3.deleteObjects(dor); logger.info("Deleted [{}] objects from AWS S3 bucket [{}]", keys.size(), awsS3Bucket.getName()); } catch (AmazonServiceException e) { throw new SystemException(AwsS3ErrorType.AWS_S3_ERROR, e) .addContextValue(AwsS3ErrorContext.OPERATION, AwsS3ErrorContext.DELETE); } catch (SdkClientException e) { throw new SystemException(AwsErrorType.AWS_SDK_CONFIGURATION_ERROR, e); } }
metrics.s3FilesDeleted.mark(1); if (deleteCount % MAX_KEYS_PER_S3_OP == 0) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteObjectsRequest); logger.info(String.format("Deleted %s files", deleteCount)); DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteObjectsRequest); logger.info(String.format("Deleted %s files", deleteCount));
.withKeys(keysToDelete) .withQuiet(true)); } else {
try { final DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName) .withKeys(part.stream().map(path -> new DeleteObjectsRequest.KeyVersion(pathPrefix + path)).collect(Collectors.toList()));
.withInitialRetryWait(100) .retryIf(e -> !(e instanceof AmazonServiceException) || !Aws.isDeterministicException((AmazonServiceException) e)) .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1);
final DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(this.awsConf.getBucketName()) .withKeys(keysToDelete) .withQuiet(false); final DeleteObjectsResult delObjRes = this.s3Client.deleteObjects(multiObjectDeleteRequest);
new DeleteObjectsRequest(bucket).withKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); if (keysToDelete.size() > 0) { DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1);
DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1);