/** * Convenience method to specify a set of keys without versions. * * @see DeleteObjectsRequest#withKeys(List) */ public DeleteObjectsRequest withKeys(String... keys) { List<KeyVersion> keyVersions = new ArrayList<KeyVersion>(keys.length); for (String key : keys) { keyVersions.add(new KeyVersion(key)); } setKeys(keyVersions); return this; }
@Override public void deleteRemoteFiles(List<Path> remotePaths) throws BackupRestoreException { if (remotePaths.isEmpty()) return; try { List<DeleteObjectsRequest.KeyVersion> keys = remotePaths .stream() .map( remotePath -> new DeleteObjectsRequest.KeyVersion( remotePath.toString())) .collect(Collectors.toList()); s3Client.deleteObjects( new DeleteObjectsRequest(getShard()).withKeys(keys).withQuiet(true)); logger.info("Deleted {} objects from S3", remotePaths.size()); } catch (Exception e) { logger.error("Error while trying to delete the objects from S3: {}", e.getMessage()); throw new BackupRestoreException(e + " while trying to delete the objects"); } }
@Override protected List<String> deleteObjects(List<String> keys) throws IOException { if (!mAlluxioConf.getBoolean(PropertyKey.UNDERFS_S3A_BULK_DELETE_ENABLED)) { return super.deleteObjects(keys); } Preconditions.checkArgument(keys != null && keys.size() <= getListingChunkLengthMax()); try { List<DeleteObjectsRequest.KeyVersion> keysToDelete = new ArrayList<>(); for (String key : keys) { keysToDelete.add(new DeleteObjectsRequest.KeyVersion(key)); } DeleteObjectsResult deletedObjectsResult = mClient.deleteObjects(new DeleteObjectsRequest(mBucketName).withKeys(keysToDelete)); List<String> deletedObjects = new ArrayList<>(); for (DeleteObjectsResult.DeletedObject deletedObject : deletedObjectsResult .getDeletedObjects()) { deletedObjects.add(deletedObject.getKey()); } return deletedObjects; } catch (AmazonClientException e) { throw new IOException(e); } }
keys.add(new KeyVersion(destinationKey));
for (File file : params.getFiles()) keys.add(new DeleteObjectsRequest.KeyVersion(file.getPath().replaceAll("\\\\", "/")));
for (File file : params.getFiles()) keys.add(new DeleteObjectsRequest.KeyVersion(file.getPath().replaceAll("\\\\", "/")));
deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion( s3ObjSumm.getKey()));
deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion( s3ObjSumm.getKey()));
List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : prevObjectListing.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
/** * Deletes the keys for the given S3 URI. The URI can represent a prefix or a single key. This method will block * on an existing async sync to S3 before running. * @param s3Uri - The S3 URI (key or prefix) to delete */ public void deleteKeys(URI s3Uri) throws InterruptedException, ExecutionException { waitForSyncToS3(); Preconditions.checkNotNull(s3Uri); Preconditions.checkArgument(SCHEME.equalsIgnoreCase(s3Uri.getScheme())); List<S3ObjectSummary> summaries = listKeys(s3Uri); if(summaries.size() > 0) { String bucket = s3Uri.getHost(); List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(); for (S3ObjectSummary summary : summaries) { keys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey())); if(keys.size() >= MAX_KEYS_PER_S3_OP) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteObjectsRequest); keys = new ArrayList<>(); } } if(keys.size() > 0) { DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucket).withKeys(keys); s3.deleteObjects(deleteObjectsRequest); } } }
/** * Delete mock parent directories which are no longer needed. * Retry policy: retrying; exceptions swallowed. * @param path path */ @Retries.RetryExceptionsSwallowed private void deleteUnnecessaryFakeDirectories(Path path) { List<DeleteObjectsRequest.KeyVersion> keysToRemove = new ArrayList<>(); while (!path.isRoot()) { String key = pathToKey(path); key = (key.endsWith("/")) ? key : (key + "/"); LOG.trace("To delete unnecessary fake directory {} for {}", key, path); keysToRemove.add(new DeleteObjectsRequest.KeyVersion(key)); path = path.getParent(); } try { removeKeys(keysToRemove, false, true); } catch(AmazonClientException | IOException e) { instrumentation.errorIgnored(); if (LOG.isDebugEnabled()) { StringBuilder sb = new StringBuilder(); for(DeleteObjectsRequest.KeyVersion kv : keysToRemove) { sb.append(kv.getKey()).append(","); } LOG.debug("While deleting keys {} ", sb.toString(), e); } } }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
@Override public void deleteAllMetadataRecords(String prefix) { checkArgument(null != prefix, "prefix should not be empty"); ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); try { Thread.currentThread().setContextClassLoader( getClass().getClassLoader()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix)); ObjectListing metaList = s3service.listObjects(listObjectsRequest); List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>(); for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) { deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey())); } if (deleteList.size() > 0) { DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket); delObjsReq.setKeys(deleteList); s3service.deleteObjects(delObjsReq); } } finally { if (contextClassLoader != null) { Thread.currentThread().setContextClassLoader(contextClassLoader); } } }
/** * Convenience method to specify a set of keys without versions. * * @see DeleteObjectsRequest#withKeys(List) */ public DeleteObjectsRequest withKeys(String... keys) { List<KeyVersion> keyVersions = new ArrayList<KeyVersion>(keys.length); for (String key : keys) { keyVersions.add(new KeyVersion(key)); } setKeys(keyVersions); return this; }
/** * Convenience method to specify a set of keys without versions. * * @see DeleteObjectsRequest#withKeys(List) */ public DeleteObjectsRequest withKeys(String... keys) { final List<KeyVersion> keyVersions = new ArrayList<KeyVersion>(keys.length); for (final String key : keys) { keyVersions.add(new KeyVersion(key)); } setKeys(keyVersions); return this; }
@Override public KeyVersion apply(long[] loc) { long z = loc[2]; long x = loc[0]; long y = loc[1]; StringBuilder sb = new StringBuilder(coordsPrefix); sb.append(z).append('/').append(x).append('/').append(y).append('.').append(extension); return new KeyVersion(sb.toString()); } }
@Override public KeyVersion apply(S3ObjectSummary input) { KeyVersion k = new KeyVersion(input.getKey()); return k; } });
@Override public KeyVersion apply(long[] loc) { long z = loc[2]; long x = loc[0]; long y = loc[1]; StringBuilder sb = new StringBuilder(coordsPrefix); sb.append(z).append('/').append(x).append('/').append(y).append('.').append(extension); return new KeyVersion(sb.toString()); } }
/** * Convenience method to specify a set of keys without versions. * * @see DeleteObjectsRequest#withKeys(List) */ public DeleteObjectsRequest withKeys(String... keys) { final List<KeyVersion> keyVersions = new ArrayList<KeyVersion>(keys.length); for (final String key : keys) { keyVersions.add(new KeyVersion(key)); } setKeys(keyVersions); return this; }
@Test public void test() { DeleteObjectsRequest req = new DeleteObjectsRequest("bucketName"); List<KeyVersion> keys = req.getKeys(); assertTrue(keys.size() == 0); keys.add(new KeyVersion("1")); assertTrue(req.getKeys().size() == 1); } }