@Override public void close() { downloadCache.close(); stagingCache.close(); }
public File get(String key) throws IOException { try { // Check if the file scheduled for async upload File staged = stagingCache.getIfPresent(key); if (staged != null && staged.exists()) { return staged; } // get from cache and download if not available return downloadCache.get(key); } catch (IOException e) { LOG.error("Error loading [{}] from cache", key); throw e; } }
@Nullable @Override public File getIfPresent(Object key) { return getIfPresent((String) key); }
@Test public void zeroCache() throws Exception { LOG.info("Started zeroCache"); cache = FileCache.build(0/* KB */, root, loader, null); closer.register(cache); File f = createFile(0, loader, cache, folder); cache.put(ID_PREFIX + 0, f); assertNull(cache.getIfPresent(ID_PREFIX + 0)); assertNull(cache.get(ID_PREFIX + 0)); assertEquals(0, cache.getStats().getMaxTotalWeight()); cache.invalidate(ID_PREFIX + 0); assertFalse(cache.containsKey(ID_PREFIX + 0)); cache.close(); LOG.info("Finished zeroCache"); }
/** * Puts the given key and file into the cache. * The file is moved to the cache. So, the original file * won't be available after this operation. It can be retrieved * using {@link #getIfPresent(String)}. * * @param key of the file * @param file to put into cache */ @Override public void put(String key, File file) { put(key, file, true); }
/** * Tests {@link FileCache#getIfPresent(Object)} when no cache. */ @Test public void getIfPresentObjectNoCache() { LOG.info("Started getIfPresentObjectNoCache"); File file = cache.getIfPresent((Object) (ID_PREFIX + 0)); assertNull(file); assertCacheStats(cache, 0, 0, 0, 0); assertEquals(1, cache.getStats().getMissCount()); LOG.info("Finished getIfPresentObjectNoCache"); }
/** * Trigger build cache on start. * @throws Exception */ @Test public void rebuild() throws Exception { LOG.info("Started rebuild"); afterExecuteLatch.await(); LOG.info("Cache built"); File f = createFile(0, loader, cache, folder); assertCache(0, cache, f); cache.close(); CountDownLatch beforeLatch = new CountDownLatch(1); CountDownLatch afterLatch = new CountDownLatch(1); afterExecuteLatch = new CountDownLatch(1); TestExecutor executor = new TestExecutor(1, beforeLatch, afterLatch, afterExecuteLatch); beforeLatch.countDown(); afterLatch.countDown(); cache = FileCache.build(4 * 1024/* bytes */, root, loader, executor); closer.register(cache); afterExecuteLatch.await(); Futures.successfulAsList((Iterable<? extends ListenableFuture<?>>) executor.futures).get(); LOG.info("Cache rebuilt"); assertCacheIfPresent(0, cache, f); assertCacheStats(cache, 1, 4 * 1024, 0, 0); LOG.info("Finished rebuild"); }
public DataStoreCacheStatsMBean getCacheStats() { return downloadCache.getStats(); }
@Test public void loadError() throws Exception { LOG.info("Started loadError"); loader = new TestErrorCacheLoader<String, InputStream>(folder.newFolder(), 8192); cache = FileCache.build(12 * 1024/* KB */, root, loader, null); closer.register(cache); createFile(0, loader, cache, folder, 12 * 1024); try { cache.get(ID_PREFIX + 0); } catch (IOException e) { } expectedEx.expect(IOException.class); cache.get(ID_PREFIX + 0); LOG.info("Finished loadError"); }
public CompositeDataStoreCache(String path, File home, long size, int uploadSplitPercentage, int uploadThreads, CacheLoader<String, InputStream> loader, final StagingUploader uploader, StatisticsProvider statsProvider, ListeningExecutorService listeningExecutor, ScheduledExecutorService scheduledExecutor /* purge scheduled executor */, ExecutorService executor /* File cache executor */, int purgeInterval /* async purge interval secs */, int stagingRetryInterval /* async retry interval secs */) { checkArgument(uploadSplitPercentage >= 0 && uploadSplitPercentage < 100, "Upload percentage should be between 0 and 100"); this.directory = new File(path); long uploadSize = (size * uploadSplitPercentage) / 100; this.stagingCache = UploadStagingCache .build(directory, home, uploadThreads, uploadSize, uploader, null, statsProvider, listeningExecutor, scheduledExecutor, purgeInterval, stagingRetryInterval); this.downloadCache = FileCache.build((size - uploadSize), directory, loader, executor); stagingCache.setDownloadCache(downloadCache); }
@Override public void invalidate(Object key) { stagingCache.invalidate((String) key); downloadCache.invalidate(key); }
/** * evict explicitly. * @throws Exception */ @Test public void evictExplicit() throws Exception { LOG.info("Started evictExplicit"); File f = createFile(0, loader, cache, folder); assertCache(0, cache, f); // trigger explicit invalidate cache.invalidate(ID_PREFIX + 0); assertFalse(cache.containsKey(ID_PREFIX + 0)); assertCacheStats(cache, 0, 0, 1, 1); LOG.info("Finished evictExplicit"); }
public static FileCache build(long maxSize /* bytes */, File root, final CacheLoader<String, InputStream> loader, @Nullable final ExecutorService executor) { if (maxSize > 0) { return new FileCache(maxSize, root, loader, executor); } return new FileCache() { @Override public void put(String key, File file) { } @Override public boolean containsKey(String key) { return false; } @Nullable @Override public File getIfPresent(String key) { return null; } @Override public File get(String key) throws IOException { return null; } @Override public void invalidate(Object key) { } @Override public DataStoreCacheStatsMBean getStats() { return new FileCacheStats(this, weigher, memWeigher, 0); } @Override public void close() { } }; }
/** * Puts the given key and file into the cache. * The file is moved to the cache. So, the original file * won't be available after this operation. It can be retrieved * using {@link #getIfPresent(String)}. * * @param key of the file * @param file to put into cache */ @Override public void put(String key, File file) { put(key, file, true); }
/** * Load and get from cache. * @throws Exception */ @Test public void add() throws Exception { LOG.info("Started add"); File f = createFile(0, loader, cache, folder); assertCache(0, cache, f); assertCacheStats(cache, 1, 4 * 1024, 1, 1); assertEquals("Memory weight different", getWeight(ID_PREFIX + 0, cache.getIfPresent(ID_PREFIX + 0)), cache.getStats().estimateCurrentMemoryWeight()); LOG.info("Finished add"); }
/** * Trigger upgrade cache on start. * @throws Exception */ @Test public void upgrade() throws Exception { LOG.info("Started upgrade"); afterExecuteLatch.await(); File f = createFile(0, loader, cache, folder); assertCache(0, cache, f); cache.close(); copyToFile(randomStream(1, 4 * 1024), getFile(ID_PREFIX + 1, root)); CountDownLatch beforeLatch = new CountDownLatch(1); CountDownLatch afterLatch = new CountDownLatch(1); afterExecuteLatch = new CountDownLatch(1); TestExecutor executor = new TestExecutor(1, beforeLatch, afterLatch, afterExecuteLatch); beforeLatch.countDown(); afterLatch.countDown(); cache = FileCache.build(4 * 1024/* bytes */, root, loader, executor); closer.register(cache); afterExecuteLatch.await(); Futures.successfulAsList((Iterable<? extends ListenableFuture<?>>) executor.futures).get(); LOG.info("Cache rebuilt"); assertCacheIfPresent(0, cache, f); assertCacheIfPresent(1, cache, copyToFile(randomStream(1, 4 * 1024), folder.newFile())); assertFalse(getFile(ID_PREFIX + 1, root).exists()); assertCacheStats(cache, 2, 8 * 1024, 0, 0); LOG.info("Finished upgrade"); }
public DataStoreCacheStatsMBean getCacheStats() { return downloadCache.getStats(); }
public CompositeDataStoreCache(String path, File home, long size, int uploadSplitPercentage, int uploadThreads, CacheLoader<String, InputStream> loader, final StagingUploader uploader, StatisticsProvider statsProvider, ListeningExecutorService listeningExecutor, ScheduledExecutorService scheduledExecutor /* purge scheduled executor */, ExecutorService executor /* File cache executor */, int purgeInterval /* async purge interval secs */, int stagingRetryInterval /* async retry interval secs */) { checkArgument(uploadSplitPercentage >= 0 && uploadSplitPercentage < 100, "Upload percentage should be between 0 and 100"); this.directory = new File(path); long uploadSize = (size * uploadSplitPercentage) / 100; this.stagingCache = UploadStagingCache .build(directory, home, uploadThreads, uploadSize, uploader, null, statsProvider, listeningExecutor, scheduledExecutor, purgeInterval, stagingRetryInterval); this.downloadCache = FileCache.build((size - uploadSize), directory, loader, executor); stagingCache.setDownloadCache(downloadCache); }
@Override public void invalidate(Object key) { stagingCache.invalidate((String) key); downloadCache.invalidate(key); }
public static FileCache build(long maxSize /* bytes */, File root, final CacheLoader<String, InputStream> loader, @Nullable final ExecutorService executor) { if (maxSize > 0) { return new FileCache(maxSize, root, loader, executor); } return new FileCache() { @Override public void put(String key, File file) { } @Override public boolean containsKey(String key) { return false; } @Nullable @Override public File getIfPresent(String key) { return null; } @Override public File get(String key) throws IOException { return null; } @Override public void invalidate(Object key) { } @Override public DataStoreCacheStatsMBean getStats() { return new FileCacheStats(this, weigher, memWeigher, 0); } @Override public void close() { } }; }