public synchronized void downloadNewFiles(File downloadDir) { log.info("Downloading new files since {}", lastMarker); BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/ BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); //Gets key within the time range specified NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages); //Download only for keys within that range for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) { log.info("Downloading file: " + blobMeta.getValue()); downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue())); lastMarker = blobMeta.getValue(); synchronized (CloudFilesManager.this) { // this is where we resume from. MarkerUtils.writeLastMarker(blobMeta.getValue()); } } log.info("Updated the last marker value as " + lastMarker); }
public synchronized boolean hasNewFiles() { // see if there are any files since lastMarker. BlobStoreContext ctx = ContextBuilder.newBuilder(provider) .credentials(user, key) .overrides(new Properties() {{ setProperty(LocationConstants.PROPERTY_ZONE, zone); }}) .buildView(BlobStoreContext.class); BlobStore store = ctx.getBlobStore(); ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker); PageSet<? extends StorageMetadata> pages = store.list(container, options); log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker); boolean emptiness = getBlobsWithinRange(pages).isEmpty(); if(emptiness) { log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME)); } else { log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME)); } return !emptiness; }
options.maxResults(maxKeys);
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
/** * @see ListContainerOptions#maxResults(int) */ public static ListContainerOptions maxResults(int maxKeys) { ListContainerOptions options = new ListContainerOptions(); return options.maxResults(maxKeys); }
public static void main(String[] args) throws Exception { BatchJCloudChunkStore st = new BatchJCloudChunkStore(); st.name = "datishpool0"; Properties p = new Properties(); p.setProperty("service-type", "b2"); st.checkAccess("a6ba900eebf7", "001c60230bbe493db4bbb8d5031a9707628790c7a9", p); System.out.println("zzz"); PageSet<? extends StorageMetadata> zips; zips = st.blobStore.list(st.name, ListContainerOptions.Builder.prefix("files/").maxResults(100)); System.out.println(zips.size()); }
@Test public void testMaxResults() { ListContainerOptions options = new ListContainerOptions(); options.maxResults(1000); assertEquals(options.getMaxResults(), Integer.valueOf(1000)); }
@Test public void testMaxResults() { ListContainerOptions options = new ListContainerOptions(); options.maxResults(1000); assertEquals(options.getMaxResults(), Integer.valueOf(1000)); }
@Test public void testTwoOptions() { ListContainerOptions options = new ListContainerOptions(); options.inDirectory("test").maxResults(1); assertEquals(options.getDir(), "test"); assertEquals(options.getMaxResults(), Integer.valueOf(1)); }
@Test public void testTwoOptions() { ListContainerOptions options = new ListContainerOptions(); options.inDirectory("test").maxResults(1); assertEquals(options.getDir(), "test"); assertEquals(options.getMaxResults(), Integer.valueOf(1)); }
public ListContainerOptions apply(ListBucketOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getDelimiter() == null) { options.recursive(); } else if (!optionsList[0].getDelimiter().equals("/")) { throw new IllegalArgumentException("only '/' is allowed as a blobstore delimiter"); } if (optionsList[0].getMarker() != null) { options.afterMarker(optionsList[0].getMarker()); } if (optionsList[0].getMaxResults() != null) { options.maxResults(optionsList[0].getMaxResults()); } if (optionsList[0].getPrefix() != null) { options.inDirectory(optionsList[0].getPrefix()); } } return options; } }
public ListContainerOptions apply(org.jclouds.atmos.options.ListOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getToken() != null) { options.afterMarker(optionsList[0].getToken()); } if (optionsList[0].getLimit() != null) { options.maxResults(optionsList[0].getLimit()); } if (optionsList[0].metaIncluded()) { options.withDetails(); } } return options; } }
public ListContainerOptions apply( org.jclouds.openstack.swift.options.ListContainerOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getPath() != null && !optionsList[0].getPath().equals("")) { options.inDirectory(optionsList[0].getPath()); } if (optionsList[0].getPrefix() != null && !optionsList[0].getPrefix().equals("")) { options.inDirectory(optionsList[0].getPrefix()); options.recursive(); } if (optionsList[0].getMarker() != null) { options.afterMarker(optionsList[0].getMarker()); } options.maxResults(optionsList[0].getMaxResults()); } return options; } }
public ListContainerOptions apply(org.jclouds.atmos.options.ListOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getToken() != null) { options.afterMarker(optionsList[0].getToken()); } if (optionsList[0].getLimit() != null) { options.maxResults(optionsList[0].getLimit()); } if (optionsList[0].metaIncluded()) { options.withDetails(); } } return options; } }
public ListContainerOptions apply(ListBlobsOptions[] optionsList) { ListContainerOptions options = new ListContainerOptions(); if (optionsList.length != 0) { if (optionsList[0].getDelimiter() == null) { options.recursive(); } else if (!optionsList[0].getDelimiter().equals("/")) { throw new IllegalArgumentException("only '/' is allowed as a blobstore delimiter"); } if (optionsList[0].getMarker() != null) { options.afterMarker(optionsList[0].getMarker()); } if (optionsList[0].getMaxResults() != null) { options.maxResults(optionsList[0].getMaxResults()); } if (optionsList[0].getPrefix() != null) { options.inDirectory(optionsList[0].getPrefix()); } if (optionsList[0].getIncludeMetadata()) { options.withDetails(); } } return options; } }