@Override public ObjectStatus[] getObjectStatuses() { StorageObject[] objects = mChunk.getObjects(); ObjectStatus[] ret = new ObjectStatus[objects.length]; for (int i = 0; i < ret.length; ++i) { ret[i] = new ObjectStatus(objects[i].getKey(), objects[i].getMd5HashAsBase64(), objects[i].getContentLength(), objects[i].getLastModifiedDate().getTime()); } return ret; }
@Override public StorageObject next() { if (!hasNext()) { throw new IllegalStateException(); } StorageObject storageObject = objectsChunk.getObjects()[objectsChunkOffset]; objectsChunkOffset++; return storageObject; }
@Override protected StorageObject[] listObjectsImpl(String bucketName, String prefix, String delimiter, long maxListingLength) throws ServiceException { return listObjectsInternal(bucketName, prefix, delimiter, maxListingLength, true, null).getObjects(); }
@Override protected StorageObject[] listObjectsImpl(String bucketName, String prefix, String delimiter, long maxListingLength) throws ServiceException { return listObjectsInternal(bucketName, prefix, delimiter, maxListingLength, true, null).getObjects(); }
public void run() { try { List allObjects = new ArrayList(); List allCommonPrefixes = new ArrayList(); do { StorageObjectsChunk chunk = storageService.listObjectsChunked( bucketName, prefix, delimiter, maxListingLength, priorLastKey); priorLastKey = chunk.getPriorLastKey(); allObjects.addAll(Arrays.asList(chunk.getObjects())); allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } while (!halted && priorLastKey != null); result = new StorageObjectsChunk( prefix, delimiter, (StorageObject[]) allObjects.toArray(new StorageObject[allObjects.size()]), (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]), null); } catch (ServiceException e) { result = e; } }
public void run() { try { List allObjects = new ArrayList(); List allCommonPrefixes = new ArrayList(); do { StorageObjectsChunk chunk = storageService.listObjectsChunked( bucketName, prefix, delimiter, maxListingLength, priorLastKey); priorLastKey = chunk.getPriorLastKey(); allObjects.addAll(Arrays.asList(chunk.getObjects())); allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } while (!halted && priorLastKey != null); result = new StorageObjectsChunk( prefix, delimiter, (StorageObject[]) allObjects.toArray(new StorageObject[allObjects.size()]), (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]), null); } catch (ServiceException e) { result = e; } }
public void run() { try { List allObjects = new ArrayList(); List allCommonPrefixes = new ArrayList(); do { StorageObjectsChunk chunk = s3Service.listObjectsChunked( bucketName, prefix, delimiter, maxListingLength, priorLastKey); priorLastKey = chunk.getPriorLastKey(); allObjects.addAll(Arrays.asList(chunk.getObjects())); allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } while (!halted && priorLastKey != null); result = new S3ObjectsChunk( prefix, delimiter, (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]), (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]), null); } catch (ServiceException se) { result = new S3ServiceException(se); } }
@Override public void s3ServiceEventPerformed(ListObjectsEvent event) { if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) { Iterator chunkIter = event.getChunkList().iterator(); while (chunkIter.hasNext()) { StorageObjectsChunk chunk = (StorageObjectsChunk) chunkIter.next(); System.out.println("Listed " + chunk.getObjects().length + " objects for sub-listing with prefix: '" + chunk.getPrefix() + "'"); allObjects.addAll(Arrays.asList(chunk.getObjects())); } } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) { s3ServiceExceptions[0] = new S3ServiceException( "Failed to list all objects in S3 bucket", event.getErrorCause()); } } });
public void run() { try { List allObjects = new ArrayList(); List allCommonPrefixes = new ArrayList(); do { StorageObjectsChunk chunk = s3Service.listObjectsChunked( bucketName, prefix, delimiter, maxListingLength, priorLastKey); priorLastKey = chunk.getPriorLastKey(); allObjects.addAll(Arrays.asList(chunk.getObjects())); allCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } while (!halted && priorLastKey != null); result = new S3ObjectsChunk( prefix, delimiter, (S3Object[]) allObjects.toArray(new S3Object[allObjects.size()]), (String[]) allCommonPrefixes.toArray(new String[allCommonPrefixes.size()]), null); } catch (ServiceException se) { result = new S3ServiceException(se); } }
@Override public void s3ServiceEventPerformed(ListObjectsEvent event) { if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) { Iterator chunkIter = event.getChunkList().iterator(); while (chunkIter.hasNext()) { StorageObjectsChunk chunk = (StorageObjectsChunk) chunkIter.next(); System.out.println("Listed " + chunk.getObjects().length + " objects for sub-listing with prefix: '" + chunk.getPrefix() + "'"); allObjects.addAll(Arrays.asList(chunk.getObjects())); } } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) { s3ServiceExceptions[0] = new S3ServiceException( "Failed to list all objects in S3 bucket", event.getErrorCause()); } } });
@Override public void iterationInit(boolean deep) { try { bs3Service = new RestS3Service(awsCredentials); StorageObjectsChunk ck = bs3Service.listObjectsChunked( this.getName(), null, null, 1000, null); this.compressedLength.set(0); this.currentLength.set(0); obj = ck.getObjects(); this.lastKey = null; objPos = 0; } catch (ServiceException e) { SDFSLogger.getLog().error("unable to initialize", e); } }
public List<ListItem> listAll(String path) { m_logger.debug("Start list all: " + path); try { List<ListItem> result = new ArrayList<>(); String priorLastKey = null; while(true) { StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "/", CHUNK_SIZE, priorLastKey); m_logger.trace("ListObjects: {}", path); inc(); StorageObject[] objects = chunk.getObjects(); for(int i = 0; i < objects.length; i++) { String key = objects[i].getKey(); if(key.endsWith("/")) key = key.substring(0, key.length() - 1); key = key.substring(path.length(), key.length()); ListItem item = new ListItem(key, objects[i].getContentLength() != 0); result.add(item); } if(chunk.isListingComplete()) break; priorLastKey = chunk.getPriorLastKey(); } return result; } catch (ServiceException e) { throw new RuntimeException(e); } }
@Override public void event(ListObjectsEvent event) { if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) { Iterator<StorageObjectsChunk> chunkIter = event.getChunkList().iterator(); while (chunkIter.hasNext()) { StorageObjectsChunk chunk = chunkIter.next(); if (log.isDebugEnabled()) { log.debug("Listed " + chunk.getObjects().length + " objects and " + chunk.getCommonPrefixes().length + " common prefixes in bucket '" + bucketName + "' using prefix=" + chunk.getPrefix() + ", delimiter=" + chunk.getDelimiter()); } allObjects.addAll(Arrays.asList(chunk.getObjects())); lastCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) { serviceExceptions[0] = new ServiceException( "Failed to list all objects in bucket", event.getErrorCause()); } } });
@Override public void event(ListObjectsEvent event) { if (ListObjectsEvent.EVENT_IN_PROGRESS == event.getEventCode()) { Iterator<StorageObjectsChunk> chunkIter = event.getChunkList().iterator(); while (chunkIter.hasNext()) { StorageObjectsChunk chunk = chunkIter.next(); if (log.isDebugEnabled()) { log.debug("Listed " + chunk.getObjects().length + " objects and " + chunk.getCommonPrefixes().length + " common prefixes in bucket '" + bucketName + "' using prefix=" + chunk.getPrefix() + ", delimiter=" + chunk.getDelimiter()); } allObjects.addAll(Arrays.asList(chunk.getObjects())); lastCommonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes())); } } else if (ListObjectsEvent.EVENT_ERROR == event.getEventCode()) { serviceExceptions[0] = new ServiceException( "Failed to list all objects in bucket", event.getErrorCause()); } } });
public static void listBucketAWS(String bucketName, String awsAccessKey, String awsSecretKey) { try { System.out .println("Listing Objects in Bucket [" + bucketName + "]"); AWSCredentials bawsCredentials = new AWSCredentials(awsAccessKey, awsSecretKey); S3Service bs3Service = new RestS3Service(bawsCredentials); StorageObjectsChunk ck = bs3Service.listObjectsChunked(bucketName, null, null, 100, null); StorageObject[] obj = ck.getObjects(); String lastKey = null; while (obj.length > 0) { for (int i = 0; i < obj.length; i++) { lastKey = obj[i].getKey(); System.out.println(lastKey); } ck = bs3Service.listObjectsChunked(bucketName, null, null, 100, lastKey); obj = ck.getObjects(); } } catch (ServiceException e) { e.printStackTrace(); } }
@Override public ObjectStatus[] getObjectStatuses() { StorageObject[] objects = mChunk.getObjects(); ObjectStatus[] ret = new ObjectStatus[objects.length]; for (int i = 0; i < ret.length; ++i) { ret[i] = new ObjectStatus(objects[i].getKey(), objects[i].getMd5HashAsBase64(), objects[i].getContentLength(), objects[i].getLastModifiedDate().getTime()); } return ret; }
public void deleteAll(String path) { try { String priorLastKey = null; while(true) { StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "?", CHUNK_SIZE, priorLastKey); m_logger.trace("ListObjects to delete: {}", path); inc(); StorageObject[] objects = chunk.getObjects(); if(objects.length == 0) break; String[] names = new String[objects.length]; for(int i = 0; i < objects.length; i++) { names[i] = objects[i].getKey(); } m_s3service.deleteMultipleObjects(BUCKET, names); m_logger.trace("DeleteObjects: {}", objects.length); // do not inc() because delete requests are not counted if(chunk.isListingComplete()) break; priorLastKey = chunk.getPriorLastKey(); } } catch (ServiceException e) { throw new RuntimeException(e); } }
@Override public boolean hasNext() { if (objectsChunk == null) { objectsChunk = listObjectsChunkedAfter(""); objectsChunkOffset = 0; } if (objectsChunk.getObjects().length <= objectsChunkOffset) { if (objectsChunk.isListingComplete()) { return false; } else { objectsChunk = listObjectsChunkedAfter(objectsChunk.getPriorLastKey()); objectsChunkOffset = 0; } } return true; }
new FileMetadata[chunk.getObjects().length]; for (int i = 0; i < fileMetadata.length; i++) { StorageObject object = chunk.getObjects()[i]; fileMetadata[i] = new FileMetadata(object.getKey(), object.getContentLength(), object.getLastModifiedDate().getTime());
if(unversioned.getObjects().length == 0) { attributes.setDuplicate(true);