private void populateMetadataWithEncryptionParams(ObjectMetadata source, ObjectMetadata destination) { Map<String, String> userMetadataSource = source.getUserMetadata(); Map<String, String> userMetadataDestination = destination.getUserMetadata(); String[] headersToCopy = { Headers.CRYPTO_CEK_ALGORITHM, Headers.CRYPTO_IV, Headers.CRYPTO_KEY, Headers.CRYPTO_KEY_V2, Headers.CRYPTO_KEYWRAP_ALGORITHM, Headers.CRYPTO_TAG_LENGTH, Headers.MATERIALS_DESCRIPTION, Headers.UNENCRYPTED_CONTENT_LENGTH, Headers.UNENCRYPTED_CONTENT_MD5 }; if (userMetadataSource != null) { if(userMetadataDestination == null){ userMetadataDestination= new HashMap<String,String>(); destination.setUserMetadata(userMetadataDestination); } String headerValue; for(String header : headersToCopy){ headerValue = userMetadataSource.get(header); if(headerValue != null){ userMetadataDestination.put(header, headerValue); } } } }
objectMetadata.setUserMetadata(userMetadata);
private void populateMetadataWithEncryptionParams(ObjectMetadata source, ObjectMetadata destination) { Map<String, String> userMetadataSource = source.getUserMetadata(); Map<String, String> userMetadataDestination = destination.getUserMetadata(); String[] headersToCopy = { Headers.CRYPTO_CEK_ALGORITHM, Headers.CRYPTO_IV, Headers.CRYPTO_KEY, Headers.CRYPTO_KEY_V2, Headers.CRYPTO_KEYWRAP_ALGORITHM, Headers.CRYPTO_TAG_LENGTH, Headers.MATERIALS_DESCRIPTION, Headers.UNENCRYPTED_CONTENT_LENGTH, Headers.UNENCRYPTED_CONTENT_MD5 }; if (userMetadataSource != null) { if (userMetadataDestination == null) { userMetadataDestination = new HashMap<String, String>(); destination.setUserMetadata(userMetadataDestination); } String headerValue; for (String header : headersToCopy) { headerValue = userMetadataSource.get(header); if (headerValue != null) { userMetadataDestination.put(header, headerValue); } } } } }
om.setUserMetadata(upload.userMetadata);
private void populateMetadataWithEncryptionParams(ObjectMetadata source, ObjectMetadata destination) { Map<String, String> userMetadataSource = source.getUserMetadata(); Map<String, String> userMetadataDestination = destination.getUserMetadata(); String[] headersToCopy = { Headers.CRYPTO_CEK_ALGORITHM, Headers.CRYPTO_IV, Headers.CRYPTO_KEY, Headers.CRYPTO_KEY_V2, Headers.CRYPTO_KEYWRAP_ALGORITHM, Headers.CRYPTO_TAG_LENGTH, Headers.MATERIALS_DESCRIPTION, Headers.UNENCRYPTED_CONTENT_LENGTH, Headers.UNENCRYPTED_CONTENT_MD5 }; if (userMetadataSource != null) { if(userMetadataDestination == null){ userMetadataDestination= new HashMap<String,String>(); destination.setUserMetadata(userMetadataDestination); } String headerValue; for(String header : headersToCopy){ headerValue = userMetadataSource.get(header); if(headerValue != null){ userMetadataDestination.put(header, headerValue); } } } }
private void populateMetadataWithEncryptionParams(ObjectMetadata source, ObjectMetadata destination) { Map<String, String> userMetadataSource = source.getUserMetadata(); Map<String, String> userMetadataDestination = destination.getUserMetadata(); String[] headersToCopy = { Headers.CRYPTO_CEK_ALGORITHM, Headers.CRYPTO_IV, Headers.CRYPTO_KEY, Headers.CRYPTO_KEY_V2, Headers.CRYPTO_KEYWRAP_ALGORITHM, Headers.CRYPTO_TAG_LENGTH, Headers.MATERIALS_DESCRIPTION, Headers.UNENCRYPTED_CONTENT_LENGTH, Headers.UNENCRYPTED_CONTENT_MD5 }; if (userMetadataSource != null) { if (userMetadataDestination == null) { userMetadataDestination = new HashMap<String, String>(); destination.setUserMetadata(userMetadataDestination); } String headerValue; for (String header : headersToCopy) { headerValue = userMetadataSource.get(header); if (headerValue != null) { userMetadataDestination.put(header, headerValue); } } } } }
@Override public void close() throws IOException { if (closed.getAndSet(true)) { return; } mBackupOutputStream.close(); LOG.debug("OutputStream for key '{}' closed. Now beginning upload", mKey); try { final ObjectMetadata om = new ObjectMetadata(); om.setContentLength(mBackupFile.length()); om.setContentType(mContentType); om.setUserMetadata(mMetadata); PutObjectRequest putObjectRequest = new PutObjectRequest(mBucketName, mKey, mBackupFile); putObjectRequest.setMetadata(om); Upload upload = transfers.upload(putObjectRequest); upload.waitForUploadResult(); } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException(e.toString()) .initCause(e); } catch (AmazonClientException e) { throw new IOException(String.format("saving output %s %s", mKey, e)); } finally { if (!mBackupFile.delete()) { LOG.warn("Could not delete temporary cos file: {}", mBackupOutputStream); } super.close(); } LOG.debug("OutputStream for key '{}' upload complete", mKey); }
@Override public void close() throws IOException { if (closed.getAndSet(true)) { return; } mBackupOutputStream.close(); LOG.debug("OutputStream for key '{}' closed. Now beginning upload", mKey); try { final ObjectMetadata om = new ObjectMetadata(); om.setContentLength(mBackupFile.length()); om.setContentType(mContentType); om.setUserMetadata(mMetadata); PutObjectRequest putObjectRequest = new PutObjectRequest(mBucketName, mKey, mBackupFile); putObjectRequest.setMetadata(om); Upload upload = transfers.upload(putObjectRequest); upload.waitForUploadResult(); } catch (InterruptedException e) { throw (InterruptedIOException) new InterruptedIOException(e.toString()) .initCause(e); } catch (AmazonClientException e) { throw new IOException(String.format("saving output %s %s", mKey, e)); } finally { if (!mBackupFile.delete()) { LOG.warn("Could not delete temporary cos file: {}", mBackupOutputStream); } super.close(); } LOG.debug("OutputStream for key '{}' upload complete", mKey); }
private void setS3ObjectUserProperty(BinaryKey binaryKey, String metadataKey, String metadataValue) throws BinaryStoreException { try { String key = binaryKey.toString(); ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, key); Map<String, String> userMetadata = metadata.getUserMetadata(); if(null != metadataValue && metadataValue.equals(userMetadata.get(metadataKey))) { return; // The key/value pair already exists in user metadata, skip update } userMetadata.put(metadataKey, metadataValue); metadata.setUserMetadata(userMetadata); // Update the object in place CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, key, bucketName, key); copyRequest.setNewObjectMetadata(metadata); s3Client.copyObject(copyRequest); } catch (AmazonClientException e) { throw new BinaryStoreException(e); } }
private void setS3ObjectUserProperty(BinaryKey binaryKey, String metadataKey, String metadataValue) throws BinaryStoreException { try { String key = binaryKey.toString(); ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, key); Map<String, String> userMetadata = metadata.getUserMetadata(); if(null != metadataValue && metadataValue.equals(userMetadata.get(metadataKey))) { return; // The key/value pair already exists in user metadata, skip update } userMetadata.put(metadataKey, metadataValue); metadata.setUserMetadata(userMetadata); // Update the object in place CopyObjectRequest copyRequest = new CopyObjectRequest(bucketName, key, bucketName, key); copyRequest.setNewObjectMetadata(metadata); s3Client.copyObject(copyRequest); } catch (AmazonClientException e) { throw new BinaryStoreException(e); } }
private com.amazonaws.services.s3.model.ObjectMetadata getAmazonMetadata(PutFileRequest request) { com.amazonaws.services.s3.model.ObjectMetadata meta = new com.amazonaws.services.s3.model.ObjectMetadata(); meta.setUserMetadata(request.getMetadata().getUserMetadata()); Map<String, Object> raw = request.getMetadata().getRawMetadata(); for (String key : raw.keySet()) { meta.setHeader(key, raw.get(key)); } meta.setContentLength(request.getFile().length()); meta.setContentEncoding(request.getEncoding()); return meta; }
private com.amazonaws.services.s3.model.ObjectMetadata getAmazonMetadata(PutDirRequest request) { com.amazonaws.services.s3.model.ObjectMetadata meta = new com.amazonaws.services.s3.model.ObjectMetadata(); meta.setUserMetadata(request.getMetadata().getUserMetadata()); Map<String, Object> raw = request.getMetadata().getRawMetadata(); for (String key : raw.keySet()) { meta.setHeader(key, raw.get(key)); } meta.setContentLength(0L); meta.setContentType(directoryContentType); meta.setContentEncoding(UTF8); return meta; }
from.setSSECustomerKeyMd5("sseCustKeyMd5"); from.setSSECustomerAlgorithm("SSECustomerAlgorithm"); from.setUserMetadata(userMetadata); from.setHeader(Headers.CONTENT_RANGE, "/9999"); from.setHeader(Headers.S3_VERSION_ID, "versionid");
@Test public void testMarkAsUnusedNoChangeNeeded() throws BinaryStoreException { ObjectMetadata objMeta = new ObjectMetadata(); Map<String, String> userMeta = new HashMap<>(); // Existing value of unused property set to true, so file is already considered // to be not used. No change should be needed. userMeta.put(s3BinaryStore.UNUSED_KEY, String.valueOf(true)); objMeta.setUserMetadata(userMeta); expect(s3Client.getObjectMetadata(eq(BUCKET), isA(String.class))) .andReturn(objMeta); replayAll(); s3BinaryStore.markAsUnused(Collections.singleton(new BinaryKey(TEST_KEY))); }
@Test public void testGetExtractedText() throws BinaryStoreException { String extractedText = "text-that-has-been-extracted"; ObjectMetadata objMeta = new ObjectMetadata(); Map<String, String> userMeta = new HashMap<>(); userMeta.put(s3BinaryStore.EXTRACTED_TEXT_KEY, extractedText); objMeta.setUserMetadata(userMeta); expect(s3Client.getObjectMetadata(BUCKET, TEST_KEY)).andReturn(objMeta); replayAll(); BinaryValue binaryValue = createBinaryValue(TEST_KEY, TEST_CONTENT); String extractValue = s3BinaryStore.getExtractedText(binaryValue); assertEquals(extractedText, extractValue); }
ObjectMetadata om = s3Service.getObjectMetadata(this.name, km); ObjectMetadata _om = new ObjectMetadata(); _om.setUserMetadata(om.getUserMetadata()); _om.addUserMetadata("lastaccessed", Long.toString(System.currentTimeMillis())); CopyObjectRequest req = new CopyObjectRequest(name, km + this.dExt, name, km + this.dExt) ObjectMetadata om = s3Service.getObjectMetadata(this.name, km); ObjectMetadata _om = new ObjectMetadata(); _om.setUserMetadata(om.getUserMetadata()); _om.addUserMetadata("lastaccessed", Long.toString(System.currentTimeMillis())); CopyObjectRequest req = new CopyObjectRequest(name, km + this.dExt, name, km + this.dExt + ".cpy")
@Test public void testMarkAsUnused() throws BinaryStoreException { ObjectMetadata objMeta = new ObjectMetadata(); Map<String, String> userMeta = new HashMap<>(); // Existing value of unused property set to false (so file is considered used) userMeta.put(s3BinaryStore.UNUSED_KEY, String.valueOf(false)); objMeta.setUserMetadata(userMeta); expect(s3Client.getObjectMetadata(eq(BUCKET), isA(String.class))) .andReturn(objMeta); Capture<CopyObjectRequest> copyRequestCapture = Capture.newInstance(); expect(s3Client.copyObject(capture(copyRequestCapture))).andReturn(null); replayAll(); s3BinaryStore.markAsUnused(Collections.singleton(new BinaryKey(TEST_KEY))); ObjectMetadata newObjMeta = copyRequestCapture.getValue().getNewObjectMetadata(); assertEquals(String.valueOf(true), newObjMeta.getUserMetadata().get(s3BinaryStore.UNUSED_KEY)); }
@Test public void testMarkAsUsed() throws BinaryStoreException { ObjectMetadata objMeta = new ObjectMetadata(); Map<String, String> userMeta = new HashMap<>(); // Existing value of unused property set to true (so file is considered not used) userMeta.put(s3BinaryStore.UNUSED_KEY, String.valueOf(true)); objMeta.setUserMetadata(userMeta); expect(s3Client.getObjectMetadata(eq(BUCKET), isA(String.class))) .andReturn(objMeta); Capture<CopyObjectRequest> copyRequestCapture = Capture.newInstance(); expect(s3Client.copyObject(capture(copyRequestCapture))).andReturn(null); replayAll(); s3BinaryStore.markAsUsed(Collections.singleton(new BinaryKey(TEST_KEY))); ObjectMetadata newObjMeta = copyRequestCapture.getValue().getNewObjectMetadata(); assertEquals(String.valueOf(false), newObjMeta.getUserMetadata().get(s3BinaryStore.UNUSED_KEY)); }
@Test public void testStoreValueExisting() throws BinaryStoreException, IOException { String valueToStore = "value-to-store"; expect(s3Client.doesObjectExist(eq(BUCKET), isA(String.class))).andReturn(true); expect(s3Client.getObjectMetadata(eq(BUCKET), isA(String.class))) .andReturn(new ObjectMetadata()); ObjectMetadata objMeta = new ObjectMetadata(); Map<String, String> userMeta = new HashMap<>(); userMeta.put(s3BinaryStore.UNUSED_KEY, String.valueOf(true)); objMeta.setUserMetadata(userMeta); Capture<CopyObjectRequest> copyRequestCapture = Capture.newInstance(); expect(s3Client.copyObject(capture(copyRequestCapture))).andReturn(null); replayAll(); s3BinaryStore.storeValue(new StringInputStream(valueToStore), true); ObjectMetadata newObjMeta = copyRequestCapture.getValue().getNewObjectMetadata(); assertEquals(String.valueOf(true), newObjMeta.getUserMetadata().get(s3BinaryStore.UNUSED_KEY)); }
private ObjectMetadata s3MetaFromSyncMeta(com.emc.ecs.sync.model.ObjectMetadata syncMeta) { com.amazonaws.services.s3.model.ObjectMetadata om = new com.amazonaws.services.s3.model.ObjectMetadata(); if (syncMeta.getCacheControl() != null) om.setCacheControl(syncMeta.getCacheControl()); if (syncMeta.getContentDisposition() != null) om.setContentDisposition(syncMeta.getContentDisposition()); if (syncMeta.getContentEncoding() != null) om.setContentEncoding(syncMeta.getContentEncoding()); om.setContentLength(syncMeta.getContentLength()); if (syncMeta.getChecksum() != null && syncMeta.getChecksum().getAlgorithm().equals("MD5")) om.setContentMD5(syncMeta.getChecksum().getValue()); if (syncMeta.getContentType() != null) om.setContentType(syncMeta.getContentType()); if (syncMeta.getHttpExpires() != null) om.setHttpExpiresDate(syncMeta.getHttpExpires()); om.setUserMetadata(formatUserMetadata(syncMeta)); if (syncMeta.getModificationTime() != null) om.setLastModified(syncMeta.getModificationTime()); return om; }