@Override public Void call() { s3.deleteObject(config.getBucketName(), identifier); return null; } }, OPERATION_DELETE_OBJECT);
@Override public String getIdentifier(String relativePath, boolean directory) { if (relativePath == null || relativePath.length() == 0) return config.getKeyPrefix(); String identifier = config.getKeyPrefix() + relativePath; // append trailing slash for directories if (directory) identifier += "/"; return identifier; }
Assert.hasText(config.getAccessKey(), "accessKey is required"); Assert.hasText(config.getSecretKey(), "secretKey is required"); Assert.hasText(config.getBucketName(), "bucketName is required"); Assert.isTrue(config.getBucketName().matches("[A-Za-z0-9._-]+"), config.getBucketName() + " is not a valid bucket name"); AWSCredentials creds = new BasicAWSCredentials(config.getAccessKey(), config.getSecretKey()); ClientConfiguration cc = new ClientConfiguration(); if (config.getProtocol() != null) cc.setProtocol(Protocol.valueOf(config.getProtocol().toString().toUpperCase())); if (config.isLegacySignatures()) cc.setSignerOverride("S3SignerType"); if (config.getSocketTimeoutMs() >= 0) cc.setSocketTimeout(config.getSocketTimeoutMs()); if (config.getHost() != null) { String portStr = ""; if (config.getPort() > 0) portStr = ":" + config.getPort(); s3.setEndpoint(config.getHost() + portStr); if (config.isDisableVHosts()) { log.info("The use of virtual hosted buckets has been DISABLED. Path style buckets will be used."); s3.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build()); boolean bucketExists = s3.doesBucketExist(config.getBucketName()); if (bucketExists && config.isIncludeVersions()) { BucketVersioningConfiguration versioningConfig = s3.getBucketVersioningConfiguration(config.getBucketName());
Assert.assertEquals("source URI mismatch", sourceUri, s3Source.getUri()); Assert.assertNull("source protocol should be null", s3Source.getProtocol()); Assert.assertEquals("source access key mismatch", sourceAccessKey, s3Source.getAccessKey()); Assert.assertEquals("source secret key mismatch", sourceSecret, s3Source.getSecretKey()); Assert.assertNull("source host should be null", s3Source.getHost()); Assert.assertEquals("source port mismatch", -1, s3Source.getPort()); Assert.assertEquals("source bucket mismatch", sourceBucket, s3Source.getBucketName()); Assert.assertNull("source keyPrefix should be null", s3Source.getKeyPrefix()); Assert.assertTrue("source includeVersions should be enabled", s3Source.isIncludeVersions()); Assert.assertEquals("target URI mismatch", targetUri, s3Target.getUri()); Assert.assertEquals("target protocol mismatch", targetProtocol, s3Target.getProtocol()); Assert.assertEquals("target access key mismatch", targetAccessKey, s3Target.getAccessKey()); Assert.assertEquals("target secret key mismatch", targetSecret, s3Target.getSecretKey()); Assert.assertEquals("target host mismatch", targetHost, s3Target.getHost()); Assert.assertEquals("target port mismatch", targetPort, s3Target.getPort()); Assert.assertEquals("target bucket mismatch", targetBucket, s3Target.getBucketName()); Assert.assertEquals("target keyPrefix mismatch", targetKeyPrefix, s3Target.getKeyPrefix()); Assert.assertTrue("target createBucket should be enabled", s3Target.isCreateBucket()); Assert.assertTrue("target disableVhost should be true", s3Target.isDisableVHosts()); Assert.assertTrue("target includeVersions should be enabled", s3Target.isIncludeVersions()); Assert.assertTrue("target legacySignatures should be enabled", s3Target.isLegacySignatures()); Assert.assertTrue("target preserveDirectories should be enabled", s3Target.isPreserveDirectories()); Assert.assertEquals("target MPU threshold mismatch", mpuThreshold, s3Target.getMpuThresholdMb()); Assert.assertEquals("target MPU part size mismatch", mpuPartSize, s3Target.getMpuPartSizeMb()); Assert.assertEquals("target MPU threads mismatch", mpuThreads, s3Target.getMpuThreadCount()); Assert.assertEquals("target socket timeout mismatch", socketTimeout, s3Target.getSocketTimeoutMs());
try { if ("".equals(config.getKeyPrefix() + object.getRelativePath())) { log.debug("Target is bucket root; skipping"); return; if (!config.isPreserveDirectories() && object.getMetadata().isDirectory()) { log.debug("Source is directory and preserveDirectories is false; skipping"); return; if (config.isIncludeVersions() && sourceVersionList != null) { ListIterator<S3ObjectVersion> sourceVersions = sourceVersionList.listIterator(); ListIterator<S3ObjectVersion> targetVersions = loadVersions(identifier).listIterator(); final CopyObjectRequest cReq = new CopyObjectRequest(config.getBucketName(), identifier, config.getBucketName(), identifier); cReq.setNewObjectMetadata(s3MetaFromSyncMeta(object.getMetadata())); time(new Function<Void>() {
S3ProgressListener progressListener = null; if (obj.getMetadata().isDirectory()) { req = new PutObjectRequest(config.getBucketName(), targetKey, new ByteArrayInputStream(new byte[0]), om); } else if (file != null) { req = new PutObjectRequest(config.getBucketName(), targetKey, file).withMetadata(om); progressListener = new ByteTransferListener(obj); } else { if (options.isMonitorPerformance()) stream = new ProgressInputStream(stream, new PerformanceListener(getWriteWindow())); req = new PutObjectRequest(config.getBucketName(), targetKey, stream, om); xferConfig.setMultipartUploadThreshold((long) config.getMpuThresholdMb() * 1024 * 1024); xferConfig.setMinimumUploadPartSize((long) config.getMpuPartSizeMb() * 1024 * 1024); TransferManager xferManager = new TransferManager(s3, Executors.newFixedThreadPool(config.getMpuThreadCount())); xferManager.setConfiguration(xferConfig);
@Override public Void call() { s3.deleteObject(config.getBucketName(), identifier); return null; } }, OPERATION_DELETE_OBJECT);
@Override public String getRelativePath(String identifier, boolean directory) { String relativePath = identifier; if (relativePath.startsWith(config.getKeyPrefix())) relativePath = relativePath.substring(config.getKeyPrefix().length()); // remove trailing slash from directories if (directory && relativePath.endsWith("/")) relativePath = relativePath.substring(0, relativePath.length() - 1); return relativePath; }
@Override public AccessControlList call() { if (versionId == null) return s3.getObjectAcl(config.getBucketName(), key); else return s3.getObjectAcl(config.getBucketName(), key, versionId); } }, OPERATION_GET_ACL);
@Override public Iterator<ObjectSummary> iterator() { return new PrefixIterator(config.getKeyPrefix()); } };
@Override public InputStream call() { GetObjectRequest request = new GetObjectRequest(config.getBucketName(), key, versionId); return s3.getObject(request).getObjectContent(); } }, OPERATION_OPEN_DATA_STREAM);
@Override public Iterator<ObjectSummary> iterator() { return new CombinedIterator<>(Arrays.asList(new PrefixIterator(config.getKeyPrefix()), new DeletedObjectIterator(config.getKeyPrefix()))); } };
@Override public ObjectMetadata call() { GetObjectMetadataRequest request = new GetObjectMetadataRequest(config.getBucketName(), key, versionId); return s3.getObjectMetadata(request); } }, OPERATION_HEAD_OBJECT);
@Override public VersionListing call() { if (fListing == null) { return s3.listVersions(config.getBucketName(), key, null, null, "/", null); } else { return s3.listNextBatchOfVersions(fListing); } } }, OPERATION_LIST_VERSIONS);
@Override public ObjectListing call() { ListObjectsRequest request = new ListObjectsRequest().withBucketName(config.getBucketName()); request.setPrefix("".equals(prefix) ? null : prefix); // Note: AWS SDK will always set encoding-type=url, but will only decode automatically if we // leave the value null return s3.listObjects(request); } }, OPERATION_LIST_OBJECTS);
@Override public Void call() { s3.deleteObjects(new DeleteObjectsRequest(config.getBucketName()).withKeys(deleteVersions)); return null; } }, OPERATION_DELETE_OBJECTS);
@Override public VersionListing call() { ListVersionsRequest request = new ListVersionsRequest().withBucketName(config.getBucketName()); request.setPrefix("".equals(prefix) ? null : prefix); // AWS SDK will always set this anyway.. this is for clarity request.setEncodingType("url"); return s3.listVersions(request); } }, OPERATION_LIST_VERSIONS);