public BlobIdTracker(String path, String repositoryId, ScheduledExecutorService scheduler, long snapshotDelaySecs, long snapshotIntervalSecs, SharedDataStore datastore) throws IOException { String root = concat(path, datastoreMeta); this.rootDir = new File(root); this.datastore = datastore; this.scheduler = scheduler; this.snapshotInterval = SECONDS.toMillis(snapshotIntervalSecs); try { forceMkdir(rootDir); prefix = fileNamePrefix + "-" + repositoryId; this.store = new BlobIdStore(rootDir, prefix); scheduler.scheduleAtFixedRate(new SnapshotJob(), SECONDS.toMillis(snapshotDelaySecs), SECONDS.toMillis(snapshotIntervalSecs), MILLISECONDS); this.deleteTracker = new ActiveDeletionTracker(rootDir, prefix); } catch (IOException e) { LOG.error("Error initializing blob tracker", e); close(); throw e; } }
@Override public int filter(GarbageCollectableBlobStore blobStore, FileLineDifferenceIterator iter, GarbageCollectorFileState fs) throws IOException { // Write the original candidates FileIOUtils.writeStrings(iter, fs.getGcCandidates(), true); // Filter the ids actively deleted BlobTrackingStore store = (BlobTrackingStore) blobStore; BlobIdTracker tracker = (BlobIdTracker) store.getTracker(); // Move the candidates identified to a temp file File candTemp = createTempFile("candTemp", null); copyFile(fs.getGcCandidates(), candTemp); Iterator<String> filter = tracker.getDeleteTracker().filter(candTemp); try { return FileIOUtils.writeStrings(filter, fs.getGcCandidates(), true); } finally { if (filter != null && filter instanceof FileLineDifferenceIterator) { ((FileLineDifferenceIterator) filter).close(); } if (candTemp != null) { candTemp.delete(); } } } },
private static Set<String> add(ActiveDeletionTracker store, List<String> ints, TemporaryFolder folder) throws IOException { File f = folder.newFile(); FileIOUtils.writeStrings(ints.iterator(), f, false); store.track(f); return Sets.newHashSet(ints); }
@Override public int filter(GarbageCollectableBlobStore blobStore, FileLineDifferenceIterator iter, GarbageCollectorFileState fs) throws IOException { // Write the original candidates FileIOUtils.writeStrings(iter, fs.getGcCandidates(), true); // Filter the ids actively deleted BlobTrackingStore store = (BlobTrackingStore) blobStore; BlobIdTracker tracker = (BlobIdTracker) store.getTracker(); // Move the candidates identified to a temp file File candTemp = createTempFile("candTemp", null); copyFile(fs.getGcCandidates(), candTemp); Iterator<String> filter = tracker.getDeleteTracker().filter(candTemp); try { return FileIOUtils.writeStrings(filter, fs.getGcCandidates(), true); } finally { if (filter != null && filter instanceof FileLineDifferenceIterator) { ((FileLineDifferenceIterator) filter).close(); } if (candTemp != null) { candTemp.delete(); } } } },
@Override public int filter(GarbageCollectableBlobStore blobStore, FileLineDifferenceIterator iter, GarbageCollectorFileState fs) throws IOException { // Write the original candidates FileIOUtils.writeStrings(iter, fs.getGcCandidates(), true); // Filter the ids actively deleted BlobTrackingStore store = (BlobTrackingStore) blobStore; BlobIdTracker tracker = (BlobIdTracker) store.getTracker(); // Move the candidates identified to a temp file File candTemp = createTempFile("candTemp", null); copyFile(fs.getGcCandidates(), candTemp); Iterator<String> filter = tracker.getDeleteTracker().filter(candTemp); try { return FileIOUtils.writeStrings(filter, fs.getGcCandidates(), true); } finally { if (filter != null && filter instanceof FileLineDifferenceIterator) { ((FileLineDifferenceIterator) filter).close(); } if (candTemp != null) { candTemp.delete(); } } } },
public BlobIdTracker(String path, String repositoryId, ScheduledExecutorService scheduler, long snapshotDelaySecs, long snapshotIntervalSecs, SharedDataStore datastore) throws IOException { String root = concat(path, datastoreMeta); this.rootDir = new File(root); this.datastore = datastore; this.scheduler = scheduler; this.snapshotInterval = SECONDS.toMillis(snapshotIntervalSecs); try { forceMkdir(rootDir); prefix = fileNamePrefix + "-" + repositoryId; this.store = new BlobIdStore(rootDir, prefix); scheduler.scheduleAtFixedRate(new SnapshotJob(), SECONDS.toMillis(snapshotDelaySecs), SECONDS.toMillis(snapshotIntervalSecs), MILLISECONDS); this.deleteTracker = new ActiveDeletionTracker(rootDir, prefix); } catch (IOException e) { LOG.error("Error initializing blob tracker", e); close(); throw e; } }
public BlobIdTracker(String path, String repositoryId, ScheduledExecutorService scheduler, long snapshotDelaySecs, long snapshotIntervalSecs, SharedDataStore datastore) throws IOException { String root = concat(path, datastoreMeta); this.rootDir = new File(root); this.datastore = datastore; this.scheduler = scheduler; this.snapshotInterval = SECONDS.toMillis(snapshotIntervalSecs); try { forceMkdir(rootDir); prefix = fileNamePrefix + "-" + repositoryId; this.store = new BlobIdStore(rootDir, prefix); scheduler.scheduleAtFixedRate(new SnapshotJob(), SECONDS.toMillis(snapshotDelaySecs), SECONDS.toMillis(snapshotIntervalSecs), MILLISECONDS); this.deleteTracker = new ActiveDeletionTracker(rootDir, prefix); } catch (IOException e) { LOG.error("Error initializing blob tracker", e); close(); throw e; } }
private ActiveDeletionTracker initTracker() throws IOException { return new ActiveDeletionTracker(root, repoId); }
@Override public void remove(File recs, Options options) throws IOException { globalMerge(); if (options == Options.ACTIVE_DELETION) { deleteTracker.track(recs); } store.removeRecords(recs); snapshot(true); }
@Override public void remove(File recs, Options options) throws IOException { globalMerge(); if (options == Options.ACTIVE_DELETION) { deleteTracker.track(recs); } store.removeRecords(recs); snapshot(true); }
@Test public void filter() throws Exception { add(tracker, range(0, 20), folder); File toFilter = create(range(7, 10), folder); Iterator<String> filtered = tracker.filter(toFilter); assertTrue("More elements after filtering", Lists.newArrayList(filtered).isEmpty()); }
@Test public void reconcileAll() throws Exception { Set<String> initAdd = add(tracker, range(0, 20), folder); List toReconcile = Lists.newArrayList(); File toFilter = create(toReconcile, folder); tracker.reconcile(toFilter); Set<String> retrieved = retrieve(tracker, folder); assertEquals("Incorrect elements after reconciliation", Sets.newHashSet(toReconcile), retrieved); }
@Override void handleRemoves(GarbageCollectableBlobStore blobStore, File removedIds, File markedRefs) throws IOException { BlobTrackingStore store = (BlobTrackingStore) blobStore; BlobIdTracker tracker = (BlobIdTracker) store.getTracker(); tracker.remove(removedIds); tracker.getDeleteTracker().reconcile(markedRefs); }
@Override void handleRemoves(GarbageCollectableBlobStore blobStore, File removedIds, File markedRefs) throws IOException { BlobTrackingStore store = (BlobTrackingStore) blobStore; BlobIdTracker tracker = (BlobIdTracker) store.getTracker(); tracker.remove(removedIds); tracker.getDeleteTracker().reconcile(markedRefs); }
@Override void handleRemoves(GarbageCollectableBlobStore blobStore, File removedIds, File markedRefs) throws IOException { BlobTrackingStore store = (BlobTrackingStore) blobStore; BlobIdTracker tracker = (BlobIdTracker) store.getTracker(); tracker.remove(removedIds); tracker.getDeleteTracker().reconcile(markedRefs); }
@Test public void reconcile() throws Exception { Set<String> initAdd = add(tracker, range(0, 20), folder); List<String> toReconcile = combine(range(7, 10), range(1, 4)); File toFilter = create(toReconcile, folder); tracker.reconcile(toFilter); Set<String> retrieved = retrieve(tracker, folder); assertEquals("Incorrect elements after reconciliation", Sets.newHashSet(toReconcile), retrieved); }
private static Set<String> retrieveActiveDeleteTracked(BlobIdTracker tracker, TemporaryFolder folder) throws IOException { File f = folder.newFile(); Set<String> retrieved = readStringsAsSet( new FileInputStream(tracker.getDeleteTracker().retrieve(f.getAbsolutePath())), false); return retrieved; }
@Test public void filterWithExtraElements() throws Exception { add(tracker, range(5, 25), folder); List<String> toFilter = combine(range(7, 10), range(0, 4)); File toFilterFile = create(toFilter, folder); Iterator<String> filtered = tracker.filter(toFilterFile); assertEquals("Incorrect elements after filtering", range(0, 4), Lists.newArrayList(filtered)); }
@Test public void filterWithNoActiveDeletion() throws Exception { File toFilter = create(range(7, 10), folder); Iterator<String> filtered = tracker.filter(toFilter); assertEquals("incorrect elements after filtering", Sets.newHashSet(range(7, 10)), Sets.newHashSet(filtered)); }
private static Set<String> retrieve(ActiveDeletionTracker store, TemporaryFolder folder) throws IOException { File f = folder.newFile(); Set<String> retrieved = readStringsAsSet( new FileInputStream(store.retrieve(f.getAbsolutePath())), false); return retrieved; }