@Override public void addMarkedStartMarker(GarbageCollectableBlobStore blobStore, String repoId) { try { ((SharedDataStore) blobStore).addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.MARKED_START_MARKER .getNameFromId(repoId)); } catch (DataStoreException e) { LOG.debug("Error creating marked time marker for repo : {}", repoId); } } },
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); Map<String, DataRecord> references = Maps.uniqueIndex(refFiles, new Function<DataRecord, String>() { @Override ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType()); Map<String, DataRecord> markers = Maps.uniqueIndex(markerFiles, new Function<DataRecord, String>() { @Override ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); String id = SharedStoreRecordType.REFERENCES.getIdFromName(repoRec.getIdentifier().toString()); GarbageCollectionRepoStats stat = new GarbageCollectionRepoStats(); stat.setRepositoryId(id);
@Override public void addMarkedStartMarker(GarbageCollectableBlobStore blobStore, String repoId, String uniqueSuffix) { try { ((SharedDataStore) blobStore).addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.MARKED_START_MARKER .getNameFromIdPrefix(repoId, uniqueSuffix)); } catch (DataStoreException e) { LOG.debug("Error creating marked time marker for repo : {}", repoId); } } },
SharedDataStore sharedDataStore = (SharedDataStore) cfg.getBlobStore(); try { sharedDataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY.getNameFromId(getOrCreateId(segmentNodeStore))); } catch (Exception e) { throw new IOException("Could not register a unique repositoryId", e);
repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore); ((SharedDataStore) blobStore).addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY.getNameFromId(repoId)); } catch (Exception e) { throw new IOException("Could not register a unique repositoryId", e);
repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore); ((SharedDataStore) blobStore).addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedDataStoreUtils.SharedStoreRecordType.REPOSITORY.getNameFromId(repoId)); } catch (Exception e) { throw new IOException("Could not register a unique repositoryId", e);
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ImmutableListMultimap<String, DataRecord> references = FluentIterable.from(refFiles).index(new Function<DataRecord, String>() { ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType()); Map<String, DataRecord> markers = Maps.uniqueIndex(markerFiles, new Function<DataRecord, String>() { @Override ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); String id = SharedStoreRecordType.REPOSITORY.getIdFromName(repoRec.getIdentifier().toString()); for(DataRecord refRec : refRecs) { String uniqueSessionId = refRec.getIdentifier().toString() .substring(SharedStoreRecordType.REFERENCES.getType().length() + 1);
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ImmutableListMultimap<String, DataRecord> references = FluentIterable.from(refFiles).index(new Function<DataRecord, String>() { ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.MARKED_START_MARKER.getType()); Map<String, DataRecord> markers = Maps.uniqueIndex(markerFiles, new Function<DataRecord, String>() { @Override ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); String id = SharedStoreRecordType.REPOSITORY.getIdFromName(repoRec.getIdentifier().toString()); for(DataRecord refRec : refRecs) { String uniqueSessionId = refRec.getIdentifier().toString() .substring(SharedStoreRecordType.REFERENCES.getType().length() + 1);
SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); DataRecord repo1 = dataStore.getMetadataRecord(SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); DataRecord repo2 = dataStore.getMetadataRecord(SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1)); DataRecord markerRec1 = dataStore.getMetadataRecord(SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1)); assertEquals( SharedStoreRecordType.MARKED_START_MARKER.getIdFromName(markerRec1.getIdentifier().toString()), repoId1); long lastModifiedMarkerRec1 = markerRec1.getLastModified(); SharedStoreRecordType.REFERENCES.getNameFromId(repoId1)); DataRecord rec1 = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId1)); long lastModifiedRec1 = rec1.getLastModified(); TimeUnit.MILLISECONDS.sleep(25); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REFERENCES.getNameFromId(repoId2)); DataRecord rec2 = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId2)); long lastModifiedRec2 = rec2.getLastModified(); assertEquals(SharedStoreRecordType.REPOSITORY.getIdFromName(repo1.getIdentifier().toString()), repoId1); assertEquals( SharedStoreRecordType.REPOSITORY.getIdFromName(repo2.getIdentifier().toString()), repoId2); assertEquals( SharedStoreRecordType.REFERENCES.getIdFromName(rec1.getIdentifier().toString()), repoId1);
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified(); LOG.trace("Earliest marker timestamp {}", earliestMarker);
data.repoIds.add(repoId1); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); data.repoIds.add(repoId2); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2));
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified(); LOG.trace("Earliest marker timestamp {}", earliestMarker);
((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()); SharedStoreRecordType.MARKED_START_MARKER.getType()); long earliestMarker = SharedDataStoreUtils.getEarliestRecord(markerFiles).getLastModified(); LOG.trace("Earliest marker timestamp {}", earliestMarker);
SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2));
@Test public void gcLongRunningBlobCollection() throws Exception { DataStoreState state = setUp(true); log.info("{} Blobs added {}", state.blobsAdded.size(), state.blobsAdded); log.info("{} Blobs should be present {}", state.blobsPresent.size(), state.blobsPresent); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); DocumentNodeStore store = mk.getNodeStore(); String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { repoId = ClusterRepositoryInfo.getOrCreateId(store); ((SharedDataStore) store.getBlobStore()).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); } TestGarbageCollector gc = new TestGarbageCollector(new DocumentBlobReferenceRetriever(store), (GarbageCollectableBlobStore) store.getBlobStore(), executor, folder.newFolder().getAbsolutePath(), 5, 5000, repoId); gc.collectGarbage(false); Set<String> existingAfterGC = iterate(); log.info("{} Blobs existing after gc {}", existingAfterGC.size(), existingAfterGC); assertTrue(Sets.difference(state.blobsPresent, existingAfterGC).isEmpty()); assertEquals(gc.additionalBlobs, Sets.symmetricDifference(state.blobsPresent, existingAfterGC)); }
@Test public void gcLongRunningBlobCollection() throws Exception { DataStoreState state = setUp(); log.info("{} Blobs added {}", state.blobsAdded.size(), state.blobsAdded); log.info("{} Blobs should be present {}", state.blobsPresent.size(), state.blobsPresent); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); String repoId = null; if (SharedDataStoreUtils.isShared(store.getBlobStore())) { repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore); ((SharedDataStore) store.getBlobStore()).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); } TestGarbageCollector gc = new TestGarbageCollector( new SegmentBlobReferenceRetriever(store), (GarbageCollectableBlobStore) store.getBlobStore(), executor, folder.newFolder().getAbsolutePath(), 5, 5000, repoId); gc.collectGarbage(false); Set<String> existingAfterGC = iterate(); log.info("{} Blobs existing after gc {}", existingAfterGC.size(), existingAfterGC); assertTrue(Sets.difference(state.blobsPresent, existingAfterGC).isEmpty()); assertEquals(gc.additionalBlobs, Sets.symmetricDifference(state.blobsPresent, existingAfterGC)); }
public Cluster(String clusterName, int clusterId, MemoryDocumentStore store) throws Exception { blobStore = new DataStoreBlobStore(createFDS(blobStoreRoot, 50)); nodeStore = builderProvider.newBuilder() .setClusterId(clusterId) .clock(clock) .setLeaseCheckMode(LeaseCheckMode.LENIENT) .setAsyncDelay(0) .setDocumentStore(store) .setBlobStore(blobStore) .getNodeStore(); repoId = ClusterRepositoryInfo.getOrCreateId(nodeStore); nodeStore.runBackgroundOperations(); ((SharedDataStore) blobStore).addMetadataRecord( new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); String trackerRoot = folder.newFolder(clusterName).getAbsolutePath(); tracker = new BlobIdTracker(trackerRoot, repoId, 86400, (SharedDataStore) blobStore); // add the tracker to the blobStore ((BlobTrackingStore) blobStore).addTracker(tracker); // initialized the GC gc = new MarkSweepGarbageCollector( new DocumentBlobReferenceRetriever(nodeStore), (GarbageCollectableBlobStore) blobStore, newSingleThreadExecutor(), folder.newFolder("gc" + clusterName).getAbsolutePath(), 5, 0, repoId); }
@Test public void externalAddOffline() throws Exception { LOG.info("In externalAddOffline"); // Close and open a new object to use the system property closer.close(); root = folder.newFolder(); File blobIdRoot = new File(root, "blobids"); blobIdRoot.mkdirs(); //Add file offline File offline = new File(blobIdRoot, "blob-offline123456.gen"); List<String> offlineLoad = range(0, 1000); FileIOUtils.writeStrings(offlineLoad.iterator(), offline, false); this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 * 60, dataStore); this.scheduler = newSingleThreadScheduledExecutor(); closer.register(tracker); closer.register(new ExecutorCloser(scheduler)); Set<String> initAdd = add(tracker, range(1001, 1005)); ScheduledFuture<?> scheduledFuture = scheduler.schedule(tracker.new SnapshotJob(), 0, TimeUnit.MILLISECONDS); scheduledFuture.get(); initAdd.addAll(offlineLoad); assertEquals(initAdd.size(), Iterators.size(tracker.get())); Set<String> retrieved = retrieve(tracker); assertEquals("Extra elements after add", initAdd, retrieved); assertTrue(read(dataStore.getAllMetadataRecords(BLOBREFERENCES.getType())).isEmpty()); }
private MarkSweepGarbageCollector getGarbageCollector() throws DataStoreException, IOException { DataStoreBlobStore blobStore = (DataStoreBlobStore) getNodeStoreComponent(BlobStore.class); FileStore fileStore = getNodeStoreComponent(FileStore.class); File fileStoreRoot = getNodeStoreComponent(FileStore.class.getName() + ":root"); if (null == garbageCollector) { String repoId = ClusterRepositoryInfo.getOrCreateId(getNodeStore()); blobStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); if (null == executor) { executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); } garbageCollector = new MarkSweepGarbageCollector( new SegmentBlobReferenceRetriever(fileStore), blobStore, executor, fileStoreRoot.getAbsolutePath(), 2048, 0, repoId ); } return garbageCollector; }
private static Map<String, String> dummyData(MemoryBlobStoreNodeStore memNodeStore, String rep2Id, NodeStore store, DataStoreBlobStore setupDataStore, File f) throws IOException, CommitFailedException, DataStoreException { List<String> list = Lists.newArrayList(); Map<String, String> idMapping = Maps.newHashMap(); NodeBuilder a = memNodeStore.getRoot().builder(); for (int i = 0; i < 2; i++) { Blob b = store.createBlob(randomStream(i+100, 18342)); Iterator<String> idIter = setupDataStore.resolveChunks(b.getContentIdentity()); while (idIter.hasNext()) { String id = idIter.next(); list.add(id); idMapping.put(id, "/d" + i); } a.child("d" + i).setProperty("x", b); } memNodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY); writeStrings(list.iterator(), f, false); sort(f); setupDataStore.addMetadataRecord(f, REFERENCES.getNameFromId(rep2Id)); return idMapping; }