public static GarbageCollectionType get(GarbageCollectableBlobStore blobStore) { if (SharedDataStoreUtils.isShared(blobStore)) { return SHARED; } return DEFAULT; }
SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles); if (unAvailRepos.isEmpty()) { ((SharedDataStore) blobStore).getAllMetadataRecords( SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified(); LOG.trace("Earliest marker timestamp {}", earliestMarker); long earliestRef = SharedDataStoreUtils.getEarliestRecord(refFiles).getLastModified(); LOG.trace("Earliest ref timestamp {}", earliestRef);
@Test public void testRefsNotAvailableAllRepos() throws Exception { File rootFolder = folder.newFolder(); dataStore = getBlobStore(rootFolder); Data data = loadData(true); // Delete one of the references file String expectedMissingRepoId = data.repoIds.get(data.repoIds.size() - 1); dataStore.deleteAllMetadataRecords( SharedStoreRecordType.REFERENCES.getNameFromId(expectedMissingRepoId)); // check if All the references from registered repositories are available Set<String> missingRepoIds = SharedDataStoreUtils .refsNotAvailableFromRepos(dataStore.getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()), dataStore.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType())); assertEquals(Sets.newHashSet(expectedMissingRepoId), missingRepoIds); }
SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles); if (unAvailRepos.isEmpty()) { ((SharedDataStore) blobStore).getAllMetadataRecords( SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified(); LOG.trace("Earliest marker timestamp {}", earliestMarker); long earliestRef = SharedDataStoreUtils.getEarliestRecord(refFiles).getLastModified(); LOG.trace("Earliest ref timestamp {}", earliestRef);
@Test public void testRefsAvailableAllRepos() throws Exception { File rootFolder = folder.newFolder(); dataStore = getBlobStore(rootFolder); loadData(true); // check if All the references from registered repositories are available Assert.assertTrue(SharedDataStoreUtils .refsNotAvailableFromRepos(dataStore.getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()), dataStore.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType())).isEmpty()); }
public static GarbageCollectionType get(GarbageCollectableBlobStore blobStore) { if (SharedDataStoreUtils.isShared(blobStore)) { return SHARED; } return DEFAULT; }
SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles); if (unAvailRepos.isEmpty()) { ((SharedDataStore) blobStore).getAllMetadataRecords( SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified(); LOG.trace("Earliest marker timestamp {}", earliestMarker); long earliestRef = SharedDataStoreUtils.getEarliestRecord(refFiles).getLastModified(); LOG.trace("Earliest ref timestamp {}", earliestRef);
public static GarbageCollectionType get(GarbageCollectableBlobStore blobStore) { if (SharedDataStoreUtils.isShared(blobStore)) { return SHARED; } return DEFAULT; }
.refsNotAvailableFromRepos(dataStore.getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()), dataStore.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType())).isEmpty()); SharedDataStoreUtils.getEarliestRecord( dataStore.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType())).getLastModified(), minRefTime); SharedDataStoreUtils.getEarliestRecord( dataStore.getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType())) .getLastModified();
public List<GarbageCollectionRepoStats> getStats() throws Exception { List<GarbageCollectionRepoStats> stats = newArrayList(); if (SharedDataStoreUtils.isShared(blobStore)) {
public List<GarbageCollectionRepoStats> getStats() throws Exception { List<GarbageCollectionRepoStats> stats = newArrayList(); if (SharedDataStoreUtils.isShared(blobStore)) {
public List<GarbageCollectionRepoStats> getStats() throws Exception { List<GarbageCollectionRepoStats> stats = newArrayList(); if (SharedDataStoreUtils.isShared(blobStore)) {
public Cluster(File root, GarbageCollectableBlobStore blobStore, NodeStore nodeStore, int seed) throws Exception { this.root = root; this.nodeStore = nodeStore; this.blobStore = blobStore; if (SharedDataStoreUtils.isShared(blobStore)) { repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore); ((SharedDataStore) blobStore).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); } referenceRetriever = ((MemoryBlobStoreNodeStore) nodeStore).getBlobReferenceRetriever(); startReferenceTime = clock.getTime(); log.info("Reference time {}", startReferenceTime); scheduledExecutor = Executors.newSingleThreadScheduledExecutor(); executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); statsProvider = new DefaultStatisticsProvider(scheduledExecutor); blobStoreState = setUp(nodeStore, blobStore, 10, 5, 100, seed); }
if (SharedDataStoreUtils.isShared(blobStore)) { String repoId = null; try {
@Test public void gcLongRunningBlobCollection() throws Exception { DataStoreState state = setUp(); log.info("{} Blobs added {}", state.blobsAdded.size(), state.blobsAdded); log.info("{} Blobs should be present {}", state.blobsPresent.size(), state.blobsPresent); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore); ((SharedDataStore) store.getBlobStore()).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); } TestGarbageCollector gc = new TestGarbageCollector( new SegmentBlobReferenceRetriever(store), (GarbageCollectableBlobStore) store.getBlobStore(), executor, folder.newFolder().getAbsolutePath(), 5, 5000, repoId); gc.collectGarbage(false); Set<String> existingAfterGC = iterate(); log.info("{} Blobs existing after gc {}", existingAfterGC.size(), existingAfterGC); assertTrue(Sets.difference(state.blobsPresent, existingAfterGC).isEmpty()); assertEquals(gc.additionalBlobs, Sets.symmetricDifference(state.blobsPresent, existingAfterGC)); }
if (SharedDataStoreUtils.isShared(blobStore)) { String repoId = null; try {
@Test public void gcLongRunningBlobCollection() throws Exception { DataStoreState state = setUp(true); log.info("{} Blobs added {}", state.blobsAdded.size(), state.blobsAdded); log.info("{} Blobs should be present {}", state.blobsPresent.size(), state.blobsPresent); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); DocumentNodeStore store = mk.getNodeStore(); String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { repoId = ClusterRepositoryInfo.getOrCreateId(store); ((SharedDataStore) store.getBlobStore()).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); } TestGarbageCollector gc = new TestGarbageCollector(new DocumentBlobReferenceRetriever(store), (GarbageCollectableBlobStore) store.getBlobStore(), executor, folder.newFolder().getAbsolutePath(), 5, 5000, repoId); gc.collectGarbage(false); Set<String> existingAfterGC = iterate(); log.info("{} Blobs existing after gc {}", existingAfterGC.size(), existingAfterGC); assertTrue(Sets.difference(state.blobsPresent, existingAfterGC).isEmpty()); assertEquals(gc.additionalBlobs, Sets.symmetricDifference(state.blobsPresent, existingAfterGC)); }
private MarkSweepGarbageCollector init(int blobGcMaxAgeInSecs, ThreadPoolExecutor executor, String root) throws Exception { DocumentNodeStore store = mk.getNodeStore(); String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { repoId = ClusterRepositoryInfo.getOrCreateId(store); ((SharedDataStore) store.getBlobStore()).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); } if (Strings.isNullOrEmpty(root)) { root = folder.newFolder().getAbsolutePath(); } MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector( new DocumentBlobReferenceRetriever(store), (GarbageCollectableBlobStore) store.getBlobStore(), executor, root, 5, blobGcMaxAgeInSecs, repoId); return gc; }
private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs, ThreadPoolExecutor executor, String root) throws Exception { String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore); ((SharedDataStore) store.getBlobStore()).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); } MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(new SegmentBlobReferenceRetriever(store), (GarbageCollectableBlobStore) store.getBlobStore(), executor, root, 2048, blobGcMaxAgeInSecs, repoId); return gc; }
if (!cfg.isSecondarySegmentStore() && cfg.hasCustomBlobStore() && isShared(cfg.getBlobStore())) { SharedDataStore sharedDataStore = (SharedDataStore) cfg.getBlobStore(); try {