@Override public ObjectStatus[] getObjectStatuses() { StorageObject[] objects = mChunk.getObjects(); ObjectStatus[] ret = new ObjectStatus[objects.length]; for (int i = 0; i < ret.length; ++i) { ret[i] = new ObjectStatus(objects[i].getKey(), objects[i].getMd5HashAsBase64(), objects[i].getContentLength(), objects[i].getLastModifiedDate().getTime()); } return ret; }
public PathAttributes toAttributes(final StorageObject object) { final PathAttributes attributes = new PathAttributes(); attributes.setSize(object.getContentLength()); final Date lastmodified = object.getLastModifiedDate(); if(lastmodified != null) { attributes.setModificationDate(lastmodified.getTime()); if(StringUtils.isNotBlank(object.getStorageClass())) { attributes.setStorageClass(object.getStorageClass()); else if(object.containsMetadata("storage-class")) { attributes.setStorageClass(object.getMetadataMap().get("storage-class").toString()); if(StringUtils.isNotBlank(object.getETag())) { attributes.setETag(object.getETag()); if(object.containsMetadata("server-side-encryption-aws-kms-key-id")) { attributes.setEncryption(new Encryption.Algorithm(object.getServerSideEncryptionAlgorithm(), object.getMetadata("server-side-encryption-aws-kms-key-id").toString()) { @Override public String getDescription() { if(null != object.getServerSideEncryptionAlgorithm()) { attributes.setEncryption(new Encryption.Algorithm(object.getServerSideEncryptionAlgorithm(), null) { @Override public String getDescription() { attributes.setChecksum(Checksum.parse(object.getETag()));
protected void putObjectWithRequestEntityImpl(String bucketName, StorageObject object, HttpEntity requestEntity, Map<String, String> requestParams) throws ServiceException { Map<String, Object> map = createObjectImpl(bucketName, object.getKey(), object.getContentType(), requestEntity, object.getMetadataMap(), requestParams, object.getAcl(), object.getStorageClass(), object.getServerSideEncryptionAlgorithm()); object.closeDataInputStream(); log.warn("Unable to close data input stream for object '" + object.getKey() + "'", e); object.replaceAllMetadata(map);
keyPath, storageObject.isDirectoryPlaceholder()); && storageObject.isDirectoryPlaceholder()) generateFileMD5Hash(file, storageObject.getKey(), progressWatcher)); if (storageObject.containsMetadata(StorageObject.METADATA_HEADER_ORIGINAL_HASH_MD5)) { objectHash = (String) storageObject.getMetadata( StorageObject.METADATA_HEADER_ORIGINAL_HASH_MD5); if (log.isDebugEnabled()) { log.debug("Object in service is encoded, using the object's original hash value for: " + storageObject.getKey()); objectHash = storageObject.getMd5HashAsBase64(); String metadataLocalFileDate = (String) storageObject.getMetadata( Constants.METADATA_JETS3T_LOCAL_FILE_DATE); objectLastModified = storageObject.getLastModifiedDate(); } else { objectLastModified = ServiceUtils log.warn("Backed-up object \"" + storageObject.getKey() + "\" and local file \"" + file.getName() + "\" have the same date but different hash values. " onlyOnClientKeys.remove(keyPath); } else {
interruptableInputStream = new InterruptableInputStream(object.getDataInputStream()); bufferedInputStream = new BufferedInputStream( new ProgressMonitoredInputStream(interruptableInputStream, progressMonitor)); if (!ServiceUtils.isEtagAlsoAnMD5Hash(object.getETag())) if (!hexMD5OfDownloadedData.equals(object.getMd5HashAsHex())) { if (log.isWarnEnabled()) { log.warn("Unable to verify MD5 hash of downloaded data against" + " ETag returned by service because ETag value \"" + object.getETag() + "\" is not an MD5 hash value" + ", for object key: " + object.getKey()); if (!hexMD5OfDownloadedData.equals(object.getETag())) { throw new ServiceException("Mismatch between MD5 hash of downloaded data (" + hexMD5OfDownloadedData + ") and ETag returned by service (" + object.getETag() + ") for object key: " + object.getKey()); } else { if (log.isDebugEnabled()) { log.debug("Object download was automatically verified, the calculated MD5 hash "+ "value matched the ETag provided by service: " + object.getKey()); object.setDataInputStream(null); object.setDataInputFile(downloadPackage.getDataFile()); String metadataLocalFileDate = (String) object.getMetadata( Constants.METADATA_JETS3T_LOCAL_FILE_DATE);
protected StorageObject putObjectImpl(String bucketName, StorageObject object) throws ServiceException { if(log.isDebugEnabled()) { log.debug("Creating Object with key " + object.getKey() + " in bucket " + bucketName); if(object.getDataInputStream() != null) { if(object.containsMetadata(StorageObject.METADATA_HEADER_CONTENT_LENGTH)) { if(log.isDebugEnabled()) { log.debug("Uploading object data with Content-Length: " + object.getContentLength()); object.getKey(), object.getDataInputStream(), object.getContentType(), object.getContentLength(), getJetS3tProperties(), isLiveMD5HashingRequired(object)); basicHttpEntity.setContent(object.getDataInputStream()); try { requestEntity = new BufferedHttpEntity(basicHttpEntity);
/** * Throws an exception if an object is null or contains a null/empty key. * @param object * @param action * the action being attempted which this assertion is applied, for debugging purposes. * @throws ServiceException */ protected void assertValidObject(StorageObject object, String action) throws ServiceException { if (object == null || object.getKey() == null || object.getKey().length() == 0) { throw new ServiceException("The action " + action + " cannot be performed with an invalid object: " + object); } }
@Override public Object clone() { StorageObject clone = new StorageObject(getKey()); clone.dataInputStream = dataInputStream; clone.acl = acl; clone.isMetadataComplete = isMetadataComplete; clone.dataInputFile = dataInputFile; clone.setOwner(this.getOwner()); clone.addAllMetadata(getMetadataMap()); return clone; }
@SuppressWarnings("deprecation") public boolean isDirectoryPlaceholder() { // Recognize "standard" directory place-holder indications used by // Amazon's AWS Console and Panic's Transmit. if (this.getKey().endsWith("/") && this.getContentLength() == 0) { return true; } // Recognize s3sync.rb directory placeholders by MD5/ETag value. if ("d66759af42f282e1ba19144df2d405d0".equals(this.getETag())) { return true; } // Recognize place-holder objects created by the Google Storage console // or S3 Organizer Firefox extension. if (this.getKey().endsWith("_$folder$") && this.getContentLength() == 0) { return true; } // Recognize legacy JetS3t directory place-holder objects, only gives // accurate results if an object's metadata is populated. if (this.getContentLength() == 0 && Mimetypes.MIMETYPE_JETS3T_DIRECTORY.equals(this.getContentType())) { return true; } return false; }
String objectKey = object.getKey(); if (forceMetadataDownload) { continue; if (!ServiceUtils.isEtagAlsoAnMD5Hash(object.getETag())) { continue; if (object.isMetadataComplete()) { if (filepath == null && object.getContentLength() == 0 && !relativeObjectKey.endsWith("/") && "d41d8cd98f00b204e9800998ecf8427e".equals(object.getETag())) if (object.getETag() != null && object.getETag().equals(fileHashAsHex)) {
public List<ListItem> listAll(String path) { m_logger.debug("Start list all: " + path); try { List<ListItem> result = new ArrayList<>(); String priorLastKey = null; while(true) { StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "/", CHUNK_SIZE, priorLastKey); m_logger.trace("ListObjects: {}", path); inc(); StorageObject[] objects = chunk.getObjects(); for(int i = 0; i < objects.length; i++) { String key = objects[i].getKey(); if(key.endsWith("/")) key = key.substring(0, key.length() - 1); key = key.substring(path.length(), key.length()); ListItem item = new ListItem(key, objects[i].getContentLength() != 0); result.add(item); } if(chunk.isListingComplete()) break; priorLastKey = chunk.getPriorLastKey(); } return result; } catch (ServiceException e) { throw new RuntimeException(e); } }
@Override public String toString() { return "StorageObject [key=" + getKey() + ", lastModified=" + getLastModifiedDate() + ", dataInputStream=" + dataInputStream + ", Metadata=" + getMetadataMap() + "]"; }
for (int i = 0; i < fileMetadata.length; i++) { StorageObject object = chunk.getObjects()[i]; fileMetadata[i] = new FileMetadata(object.getKey(), object.getContentLength(), object.getLastModifiedDate().getTime());
@Override public InputStream openInputStream() throws IOException { try { synchronized (inputStreamOpener) { if (streamAcquired) { return storageObject.getDataInputStream(); } // lazily promote to full GET storageObject = s3Client.getObject(s3Obj.getBucketName(), s3Obj.getKey()); final InputStream stream = storageObject.getDataInputStream(); streamAcquired = true; return stream; } } catch (ServiceException e) { throw new IOE(e, "Could not load S3 URI [%s]", uri); } }
lastKey = obj[objPos].getKey(); if (lastKey.equals("bucketinfo")) { objPos++; if (obj[objPos].containsMetadata("encrypt")) { encrypt = Boolean.parseBoolean((String) obj[objPos] .getMetadata("encrypt")); obj[objPos].getKey(), encrypt), 0); if (obj[objPos].containsMetadata("size")) { chk.cLen = Integer.parseInt((String) obj[objPos] .getMetadata("size")); this.currentLength.addAndGet(chk.cLen); if (obj[objPos].containsMetadata("compressedsize")) { int cl = Integer.parseInt((String) obj[objPos] .getMetadata("compressedsize")); this.compressedLength.addAndGet(cl);
@Override public FileMetadata retrieveMetadata(String key) throws IOException { StorageObject object = null; try { LOG.debug("Getting metadata for key: {} from bucket: {}", key, bucket.getName()); object = s3Service.getObjectDetails(bucket.getName(), key); return new FileMetadata(key, object.getContentLength(), object.getLastModifiedDate().getTime()); } catch (ServiceException e) { try { // process handleException(e, key); return null; } catch (FileNotFoundException fnfe) { // and downgrade missing files return null; } } finally { if (object != null) { object.closeDataInputStream(); } } }
@Override public InputStream openStream() throws IOException { try { final long start; final long end = objectDetails.getContentLength() - 1; if (offset > 0 && offset < objectDetails.getContentLength()) { start = offset; } else if (offset < 0 && (-1 * offset) < objectDetails.getContentLength()) { start = objectDetails.getContentLength() + offset; } else { start = 0; } return service.getObject( config.getS3Bucket(), taskKey, null, null, new String[]{objectDetails.getETag()}, null, start, end ).getDataInputStream(); } catch (ServiceException e) { throw new IOException(e); } } }
null == part.getLastModifiedDate() ? new Date(System.currentTimeMillis()) : part.getLastModifiedDate(), null == part.getETag() ? StringUtils.EMPTY : part.getETag(), part.getContentLength());
if(!expectedETag.equals(uploadedObject.getETag())) { throw new ServiceException("Mismatch between MD5 hash of uploaded data (" + expectedETag + ") and ETag returned by S3 (" + uploadedObject.getETag() + ") for object key: " + uploadedObject.getKey()); "value matched the ETag returned by S3: " + uploadedObject.getKey());
if (object.isDirectoryPlaceholder()) { return null; ("gzip".equalsIgnoreCase(object.getContentEncoding()) || object.containsMetadata(Constants.METADATA_JETS3T_COMPRESSED))) && object.containsMetadata(Constants.METADATA_JETS3T_CRYPTO_ALGORITHM)) String algorithm = (String) object.getMetadata( Constants.METADATA_JETS3T_CRYPTO_ALGORITHM); String version = (String) object.getMetadata( Constants.METADATA_JETS3T_CRYPTO_VERSION); if (version == null) {