Refine search
private static void deleteByFullPathPrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); String[] keys = listByFullPathPrefix(bucket, s3prefix).toArray(new String[] {}); if (keys.length > 0) { DeleteObjectsRequest request = new DeleteObjectsRequest(bucket).withKeys(keys); AmazonS3ClientBuilder.defaultClient().deleteObjects(request); } }
/** * Sets the list of keys to delete from this bucket, clearing any existing * list of keys. * * @param keys * The list of keys to delete from this bucket * * @return this, to chain multiple calls togethers. */ public DeleteObjectsRequest withKeys(List<KeyVersion> keys) { setKeys(keys); return this; }
/** * Converts the specified {@link DeleteObjectsRequest} object to an XML fragment that * can be sent to Amazon S3. * * @param rq * The {@link DeleteObjectsRequest} */ public byte[] convertToXmlByteArray(DeleteObjectsRequest rq) throws SdkClientException { XmlWriter xml = new XmlWriter(); xml.start("Delete"); if ( rq.getQuiet() ) { xml.start("Quiet").value("true").end(); } for (KeyVersion keyVersion : rq.getKeys()) { writeKeyVersion(xml, keyVersion); } xml.end(); return xml.getBytes(); }
@Override public void deleteRemoteFiles(List<Path> remotePaths) throws BackupRestoreException { if (remotePaths.isEmpty()) return; try { List<DeleteObjectsRequest.KeyVersion> keys = remotePaths .stream() .map( remotePath -> new DeleteObjectsRequest.KeyVersion( remotePath.toString())) .collect(Collectors.toList()); s3Client.deleteObjects( new DeleteObjectsRequest(getShard()).withKeys(keys).withQuiet(true)); logger.info("Deleted {} objects from S3", remotePaths.size()); } catch (Exception e) { logger.error("Error while trying to delete the objects from S3: {}", e.getMessage()); throw new BackupRestoreException(e + " while trying to delete the objects"); } }
@Override protected List<String> deleteObjects(List<String> keys) throws IOException { if (!mAlluxioConf.getBoolean(PropertyKey.UNDERFS_S3A_BULK_DELETE_ENABLED)) { return super.deleteObjects(keys); } Preconditions.checkArgument(keys != null && keys.size() <= getListingChunkLengthMax()); try { List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>(); for (String key : keys) { keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); } DeleteObjectsResult deletedObjectsResult = mClient.deleteObjects(new DeleteObjectsRequest(mBucketName).withKeys(keysToDelete)); List<String> deletedObjects = new ArrayList<>(); for (DeleteObjectsResult.DeletedObject deletedObject : deletedObjectsResult .getDeletedObjects()) { deletedObjects.add(deletedObject.getKey()); } return deletedObjects; } catch (AmazonClientException e) { throw new IOException(e); } }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
AmazonS3Client s3service = Utils.openService(props); TransferManager tmx = new TransferManager(s3service); if (s3service.doesBucketExist(bucket)) { for (int i = 0; i < 4; i++) { tmx.abortMultipartUploads(bucket, date); ObjectListing prevObjectListing = s3service.listObjects(bucket); while (prevObjectListing != null ) { List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion( s3ObjSumm.getKey())); DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest( bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq);
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(s3BucketName); multiObjectDeleteRequest.setKeys(keysSubList); try LOGGER.info("s3Key=\"{}\" s3VersionId=\"{}\"", keyVersion.getKey(), keyVersion.getVersion());
List<KeyVersion> keys = Lists.transform(locations, tileToKey); DeleteObjectsRequest req = new DeleteObjectsRequest(bucketName); req.setQuiet(true); req.setKeys(keys); conn.deleteObjects(req);
@Override public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) { deleteObjectsRequest = beforeClientExecution(deleteObjectsRequest); Request<DeleteObjectsRequest> request = createRequest(deleteObjectsRequest.getBucketName(), null, deleteObjectsRequest, HttpMethodName.POST); request.addHandlerContext(HandlerContextKey.OPERATION_NAME, "DeleteObjects"); request.addParameter("delete", null); if (deleteObjectsRequest.getBypassGovernanceRetention()) { request.addHeader(Headers.BYPASS_GOVERNANCE_RETENTION, "true"); if ( deleteObjectsRequest.getMfa() != null ) { populateRequestWithMfaDetails(request, deleteObjectsRequest.getMfa()); populateRequesterPaysHeader(request, deleteObjectsRequest.isRequesterPays()); new S3RequesterChargedHeaderHandler<DeleteObjectsResponse>()); DeleteObjectsResponse response = invoke(request, responseHandler, deleteObjectsRequest.getBucketName(), null);
for (S3ObjectSummary so : partition) { String key = so.getKey(); keys.add(new KeyVersion(key)); DeleteObjectsRequest deleteReq = new DeleteObjectsRequest(bucketName); deleteReq.setQuiet(true); deleteReq.setKeys(keys);
public void delete() { checkState(isConfigured(), "client not configured."); if (temporaryPrefix == null) { return; } Iterable<S3ObjectSummary> objects = S3Objects.withPrefix(s3, bucket, temporaryPrefix); Iterable<List<S3ObjectSummary>> partition = Iterables.partition(objects, 1000); for (List<S3ObjectSummary> os : partition) { List<KeyVersion> keys = Lists.transform( os, new Function<S3ObjectSummary, KeyVersion>() { @Override public KeyVersion apply(S3ObjectSummary input) { KeyVersion k = new KeyVersion(input.getKey()); return k; } }); DeleteObjectsRequest deleteRequest = new DeleteObjectsRequest(bucket); deleteRequest.setKeys(keys); s3.deleteObjects(deleteRequest); } }
@Test public void test() { DeleteObjectsRequest req = new DeleteObjectsRequest("bucketName"); List<KeyVersion> keys = req.getKeys(); assertTrue(keys.size() == 0); keys.add(new KeyVersion("1")); assertTrue(req.getKeys().size() == 1); } }
List<KeyVersion> keys = new ArrayList<KeyVersion>(); keys.add(new KeyVersion("dev/3123/compressedc1eac77b-9c38-4036-9770-34a77a163bb0.jpeg")); keys.add(new KeyVersion("dev/3123/compressedb52adf1e-5155-48b6-9051-bb679601f5ee.jpeg")); DeleteObjectsRequest request = new DeleteObjectsRequest("mubucketname").withKeys(keys); DeleteObjectsResult result = s3client.deleteObjects(request);
@Override public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest) throws AmazonClientException, AmazonServiceException { maybeFail(); for (DeleteObjectsRequest.KeyVersion keyVersion : deleteObjectsRequest.getKeys()) { registerDeleteObject(keyVersion.getKey(), deleteObjectsRequest.getBucketName()); } return super.deleteObjects(deleteObjectsRequest); }
@Override public DeleteObjectsResult deleteObjects(DeleteObjectsRequest deleteObjectsRequest, AmazonS3 s3Client) { LOGGER.debug("deleteObjects(): deleteObjectRequest.getBucketName() = " + deleteObjectsRequest.getBucketName() + ", deleteObjectRequest.getKeys() = " + deleteObjectsRequest.getKeys()); List<DeletedObject> deletedObjects = new ArrayList<>(); MockS3Bucket mockS3Bucket = mockS3Buckets.get(deleteObjectsRequest.getBucketName()); for (KeyVersion keyVersion : deleteObjectsRequest.getKeys()) { String s3ObjectKey = keyVersion.getKey(); String s3ObjectVersion = keyVersion.getVersion(); String s3ObjectKeyVersion = s3ObjectKey + (s3ObjectVersion != null ? s3ObjectVersion : ""); mockS3Bucket.getObjects().remove(s3ObjectKey); if (mockS3Bucket.getVersions().remove(s3ObjectKeyVersion) != null) { DeletedObject deletedObject = new DeletedObject(); deletedObject.setKey(s3ObjectKey); deletedObject.setVersionId(s3ObjectVersion); deletedObjects.add(deletedObject); } } return new DeleteObjectsResult(deletedObjects); }
/** * Sets the quiet element for this request. When true, only errors will be * returned in the service response. * * @return this, to chain multiple calls together. */ public DeleteObjectsRequest withQuiet(boolean quiet) { this.setQuiet(quiet); return this; }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
AmazonS3Client s3service = Utils.openService(props); TransferManager tmx = new TransferManager(s3service); if (s3service.doesBucketExist(bucket)) { for (int i = 0; i < 4; i++) { tmx.abortMultipartUploads(bucket, date); ObjectListing prevObjectListing = s3service.listObjects(bucket); while (prevObjectListing != null ) { List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion( s3ObjSumm.getKey())); DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest( bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq);
DeleteObjectsRequest multiObjectDeleteRequest = new DeleteObjectsRequest(s3BucketName); multiObjectDeleteRequest.setKeys(keysSubList); try LOGGER.info("s3Key=\"{}\" s3VersionId=\"{}\"", keyVersion.getKey(), keyVersion.getVersion());