public CompositeDataStoreCache(String path, File home, long size, int uploadSplitPercentage, int uploadThreads, CacheLoader<String, InputStream> loader, final StagingUploader uploader, StatisticsProvider statsProvider, ListeningExecutorService listeningExecutor, ScheduledExecutorService scheduledExecutor /* purge scheduled executor */, ExecutorService executor /* File cache executor */, int purgeInterval /* async purge interval secs */, int stagingRetryInterval /* async retry interval secs */) { checkArgument(uploadSplitPercentage >= 0 && uploadSplitPercentage < 100, "Upload percentage should be between 0 and 100"); this.directory = new File(path); long uploadSize = (size * uploadSplitPercentage) / 100; this.stagingCache = UploadStagingCache .build(directory, home, uploadThreads, uploadSize, uploader, null, statsProvider, listeningExecutor, scheduledExecutor, purgeInterval, stagingRetryInterval); this.downloadCache = FileCache.build((size - uploadSize), directory, loader, executor); stagingCache.setDownloadCache(downloadCache); }
@Nullable public File getIfPresent(String key) { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } return downloadCache.getIfPresent(key); }
public DataStoreCacheStatsMBean getStagingCacheStats() { return stagingCache.getStats(); }
@Test public void testZeroCache() throws IOException { stagingCache = UploadStagingCache.build(root, null, 1/*threads*/, 0 /* bytes */, uploader, null/*cache*/, statsProvider, executor, null, 3000, 6000); closer.register(stagingCache); File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 0, f); assertFalse(future.isPresent()); assertNull(stagingCache.getIfPresent(ID_PREFIX + 0)); assertEquals(0, Iterators.size(stagingCache.getAllIdentifiers())); assertEquals(0, stagingCache.getStats().getMaxTotalWeight()); }
@Test public void testDefaultStatsProvider() throws Exception { stagingCache = UploadStagingCache.build(root, null, 1/*threads*/, 8 * 1024 /* bytes */, uploader, null/*cache*/, null, executor, null, 3000, 6000); // add load File f = copyToFile(randomStream(0, 4 * 1024), folder.newFile()); Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 0, f); assertTrue(future.isPresent()); assertNotNull(stagingCache.getIfPresent(ID_PREFIX + 0)); assertCacheStats(stagingCache, 1, 4 * 1024, 1, 1); }
/** * Retrieve without adding. * @throws Exception */ @Test public void testGetNoAdd() throws Exception { File ret = stagingCache.getIfPresent(ID_PREFIX + 0); // assert no file assertNull(ret); assertEquals(1, stagingCache.getStats().getLoadCount()); assertCacheStats(stagingCache, 0, 0, 0, 0); }
/** * Error in putting file to stage. * @throws Exception */ @Test public void testPutMoveFileError() throws Exception { File empty = new File(folder.getRoot(), String.valueOf(System.currentTimeMillis())); assertFalse(empty.exists()); Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 0, empty); // assert no file assertFalse(future.isPresent()); assertEquals(1, stagingCache.getStats().getMissCount()); assertCacheStats(stagingCache, 0, 0, 0, 1); }
/** * Invalidate after staging before upload. * @throws Exception */ @Test public void testInvalidate() throws Exception { // add load List<ListenableFuture<Integer>> futures = put(folder); // Check invalidate stagingCache.invalidate(ID_PREFIX + 0); File file = stagingCache.getIfPresent(ID_PREFIX + 0); assertNull(file); //start taskLatch.countDown(); callbackLatch.countDown(); waitFinish(futures); assertCacheStats(stagingCache, 0, 0, 1, 1); // Should not return anything file = stagingCache.getIfPresent(ID_PREFIX + 0); assertNull(file); }
public boolean stage(String key, File file) { return stagingCache.put(key, file).isPresent(); }
/** * Stage file when cache full. * @throws Exception */ @Test public void testCacheFullAdd() throws Exception { // initialize cache to have restricted size stagingCache = UploadStagingCache.build(root, null, 1/*threads*/, 4 * 1024 /* bytes */, uploader, null/*cache*/, statsProvider, executor, null, 3000, 6000); closer.register(stagingCache); // add load List<ListenableFuture<Integer>> futures = put(folder); // Add another load File f2 = copyToFile(randomStream(1, 4 * 1024), folder.newFile()); Optional<SettableFuture<Integer>> future2 = stagingCache.put(ID_PREFIX + 1, f2); assertFalse(future2.isPresent()); //start taskLatch.countDown(); callbackLatch.countDown(); assertFuture(futures, 0); // Try 2nd upload again Optional<SettableFuture<Integer>> future = stagingCache.put(ID_PREFIX + 1, f2); futures = Lists.newArrayList(); if (future.isPresent()) { futures.add(future.get()); } assertFuture(futures, 1); assertCacheStats(stagingCache, 0, 0, 2, 3); }
|| currentSize.addAndGet(length) <= size) && !attic.containsKey(id) && existsOrNotExistsMoveFile(input, uploadFile, currentSize, length) && map.putIfAbsent(id, uploadFile) == null ) { cacheStats.incrementMemSize(memWeigher.weigh(id, uploadFile)); return Optional.of(stage(id, uploadFile)); } catch (Exception e) { LOG.info("Error moving file to staging", e);
/** * In rare cases may include some duplicates in cases where async staged uploads complete * during iteration. * * @return Iterator over all ids available * @throws DataStoreException */ @Override public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException { return Iterators.concat(Iterators.transform(cache.getStagingCache().getAllIdentifiers(), new Function<String, DataIdentifier>() { @Nullable @Override public DataIdentifier apply(@Nullable String id) { return new DataIdentifier(id); } }), backend.getAllIdentifiers()); }
@Override public void invalidate(Object key) { stagingCache.invalidate((String) key); downloadCache.invalidate(key); }
this.downloadCache = cache; build(home, dir);
/** * Invalidate called externally. * @param key to invalidate */ protected void invalidate(String key) { // Check if not already scheduled for deletion if (!attic.containsKey(key) && map.containsKey(key)) { try { LOG.debug("Invalidating [{}]", key); File toBeDeleted = map.get(key); deleteInternal(key, toBeDeleted); map.remove(key, toBeDeleted); } catch (IOException e) { LOG.warn("Could not delete file from staging", e); } } }
@Override public void close() { downloadCache.close(); stagingCache.close(); }
public static UploadStagingCache build(File dir, File home, int uploadThreads, long size /* bytes */, StagingUploader uploader, @Nullable FileCache cache, StatisticsProvider statisticsProvider, @Nullable ListeningExecutorService executor, @Nullable ScheduledExecutorService scheduledExecutor, int purgeInterval /* secs */, int retryInterval /* secs */) { if (size > 0) { return new UploadStagingCache(dir, home, uploadThreads, size, uploader, cache, statisticsProvider, executor, scheduledExecutor, purgeInterval, retryInterval); } return new UploadStagingCache() { @Override public Optional<SettableFuture<Integer>> put(String id, File input) { return Optional.absent(); } @Override protected void invalidate(String key) { } @Override protected Iterator<String> getAllIdentifiers() { return Collections.emptyIterator(); } @Nullable @Override public File getIfPresent(String key) { return null; } @Override public DataStoreCacheStatsMBean getStats() { return new StagingCacheStats(this, StatisticsProvider.NOOP, 0); } @Override public void close() { } }; }
File ret = stagingCache.getIfPresent(ID_PREFIX + 0); assertTrue(Files.equal(copyToFile(randomStream(0, 4 * 1024), folder.newFile()), ret)); assertEquals(1, stagingCache.getStats().getLoadCount()); assertEquals(1, stagingCache.getStats().getLoadSuccessCount()); assertCacheStats(stagingCache, 1, 4 * 1024, 1, 1); ret = stagingCache.getIfPresent(ID_PREFIX + 0); assertNull(ret); assertTrue(Files.equal(copyToFile(randomStream(0, 4 * 1024), folder.newFile()),
public boolean stage(String key, File file) { return stagingCache.put(key, file).isPresent(); }
|| currentSize.addAndGet(length) <= size) && !attic.containsKey(id) && existsOrNotExistsMoveFile(input, uploadFile, currentSize, length) && map.putIfAbsent(id, uploadFile) == null ) { cacheStats.incrementMemSize(memWeigher.weigh(id, uploadFile)); return Optional.of(stage(id, uploadFile)); } catch (Exception e) { LOG.info("Error moving file to staging", e);