@Override protected List<String> deleteObjects(List<String> keys) throws IOException { if (!mAlluxioConf.getBoolean(PropertyKey.UNDERFS_S3A_BULK_DELETE_ENABLED)) { return super.deleteObjects(keys); } Preconditions.checkArgument(keys != null && keys.size() <= getListingChunkLengthMax()); try { List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>(); for (String key : keys) { keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); } DeleteObjectsResult deletedObjectsResult = mClient.deleteObjects(new DeleteObjectsRequest(mBucketName).withKeys(keysToDelete)); List<String> deletedObjects = new ArrayList<>(); for (DeleteObjectsResult.DeletedObject deletedObject : deletedObjectsResult .getDeletedObjects()) { deletedObjects.add(deletedObject.getKey()); } return deletedObjects; } catch (AmazonClientException e) { throw new IOException(e); } }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList( startIndex, endIndex))); DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq); LOG.info( "Records[{}] deleted in datastore from index [{}] to [{}]",
delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList( startIndex, endIndex))); DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq); LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1));
delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList( startIndex, endIndex))); DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq); LOG.info("Records[{}] deleted in datastore from index [{}] to [{}]", dobjs.getDeletedObjects().size(), startIndex, (endIndex - 1));
delObjsReq.setKeys(Collections.unmodifiableList(deleteList.subList( startIndex, endIndex))); DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq); LOG.info( "Records[{}] deleted in datastore from index [{}] to [{}]",
bucket); delObjsReq.setKeys(deleteList); DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq); if (dobjs.getDeletedObjects().size() != deleteList.size()) { throw new DataStoreException(
bucket); delObjsReq.setKeys(deleteList); DeleteObjectsResult dobjs = s3service.deleteObjects(delObjsReq); if (dobjs.getDeletedObjects().size() != deleteList.size()) { throw new DataStoreException(
@Override public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) throws AmazonClientException, AmazonServiceException { maybeFail(); for (DeleteObjectsRequest.KeyVersion keyVersion : deleteObjectsRequest.getKeys()) { registerDeleteObject(keyVersion.getKey(), deleteObjectsRequest.getBucketName()); } return super.deleteObjects(deleteObjectsRequest); }
bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq);
bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq);
DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); keys.clear(); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1);
.withInitialRetryWait(100) .retryIf(e -> !(e instanceof AmazonServiceException) || !Aws.isDeterministicException((AmazonServiceException) e)) .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); keys.clear(); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1);
DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1); keysToDelete.clear(); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket).withKeys(keysToDelete); s3.deleteObjects(deleteRequest); statistics.incrementWriteOps(1);
public void delete() { checkState(isConfigured(), "client not configured."); if (temporaryPrefix == null) { return; } Iterable<S3ObjectSummary> objects = S3Objects.withPrefix(s3, bucket, temporaryPrefix); Iterable<List<S3ObjectSummary>> partition = Iterables.partition(objects, 1000); for (List<S3ObjectSummary> os : partition) { List<KeyVersion> keys = Lists.transform( os, new Function<S3ObjectSummary, KeyVersion>() { @Override public KeyVersion apply(S3ObjectSummary input) { KeyVersion k = new KeyVersion(input.getKey()); return k; } }); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket); deleteRequest.setKeys(keys); s3.deleteObjects(deleteRequest); } }
req.setQuiet(true); req.setKeys(keys); conn.deleteObjects(req);
req.setQuiet(true); req.setKeys(keys); conn.deleteObjects(req);
DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq);