@Override public BlobStore setUp() { String className = System.getProperty("dataStore"); checkNotNull(className, "No system property named 'dataStore' defined"); try { dataStore = Class.forName(className).asSubclass(DataStore.class).newInstance(); config = getConfig(); configure(dataStore, config); dataStore = configureIfCloudDataStore(className, dataStore, config, unique.toLowerCase(), statisticsProvider); storeDir = new File(basedir, unique); dataStore.init(storeDir.getAbsolutePath()); blobStore = new DataStoreBlobStore(dataStore, true, fdsCacheInMB); configure(blobStore); return blobStore; } catch (Exception e) { throw new IllegalStateException("Cannot instantiate DataStore " + className, e); } }
@Override public void close() throws IOException { if (dataStore != null) { try { dataStore.close(); } catch (DataStoreException e) { throw new IOException(e); } } }
@Override public String writeBlob(String tempFileName) throws IOException { File file = new File(tempFileName); InputStream in = null; try { in = new FileInputStream(file); return writeBlob(in); } finally { closeQuietly(in); FileUtils.forceDelete(file); } }
@Override public BlobStore setUp() { fds = new FileDataStore(); fds.setMinRecordLength(4092); storeDir = new File(basedir, unique); fds.init(storeDir.getAbsolutePath()); configure(fds); DataStoreBlobStore bs = new DataStoreBlobStore(fds, true, fdsCacheInMB); bs.setBlobStatsCollector(new BlobStoreStats(statisticsProvider)); configure(bs); return bs; }
@Test public void testInlineBinary() throws DataStoreException, IOException { int maxInlineSize = 300; DataStore mockedDS = mock(DataStore.class); when(mockedDS.getMinRecordLength()).thenReturn(maxInlineSize); DataStoreBlobStore ds = new DataStoreBlobStore(mockedDS); byte[] data = new byte[maxInlineSize]; new Random().nextBytes(data); DataRecord dr = ds.addRecord(new ByteArrayInputStream(data)); assertTrue(InMemoryDataRecord.isInstance(dr.getIdentifier().toString())); assertTrue(IOUtils.contentEquals(new ByteArrayInputStream(data), dr.getStream())); assertTrue(IOUtils.contentEquals(new ByteArrayInputStream(data), new BlobStoreInputStream(ds, dr.getIdentifier().toString(), 0))); assertEquals(dr, ds.getRecordIfStored(dr.getIdentifier())); assertEquals(dr, ds.getRecord(dr.getIdentifier())); //Check for BlobStore methods assertEquals(maxInlineSize, ds.getBlobLength(dr.getIdentifier().toString())); assertEquals(dr.getIdentifier().toString(), BlobId.of(ds.writeBlob(new ByteArrayInputStream(data))).blobId); }
@Test public void inMemoryRecord() throws Exception{ File fdsDir = temporaryFolder.newFolder(); FileDataStore fds = DataStoreUtils.createFDS(fdsDir, 10000); DataStoreBlobStore dbs = new DataStoreBlobStore(fds); ByteArrayInputStream is = new ByteArrayInputStream("".getBytes()); String blobId = dbs.writeBlob(is); File writerDir = temporaryFolder.newFolder(); PreExtractedTextProvider textProvider = new DataStoreTextWriter(writerDir, true); assertNull(textProvider.getText("/content", new BlobStoreBlob(dbs, blobId))); }
@Test public void testAddOnTrackError() throws Exception { int maxInlineSize = 300; byte[] data = new byte[maxInlineSize]; new Random().nextBytes(data); DataStore mockedDS = mock(DataStore.class); when(mockedDS.getMinRecordLength()).thenReturn(maxInlineSize); DataStoreBlobStore ds = new DataStoreBlobStore(mockedDS); BlobIdTracker mockedTracker = mock(BlobIdTracker.class); doThrow(new IOException("Mocking tracking error")).when(mockedTracker).add(any(String.class)); ds.addTracker(mockedTracker); String id = ds.writeBlob(new ByteArrayInputStream(data)); assertTrue(IOUtils.contentEquals(new ByteArrayInputStream(data), ds.getInputStream(id))); }
@Test public void testConsistency() throws Exception { File dump = temporaryFolder.newFolder(); File repoHome = temporaryFolder.newFolder(); Random rand = new Random(); String deletedBlobId = Iterables.get(blobsAdded, rand.nextInt(blobsAdded.size())); blobsAdded.remove(deletedBlobId); long count = setupDataStore.countDeleteChunks(ImmutableList.of(deletedBlobId), 0); assertEquals(1, count); setupDataStore.close(); testAllParams(dump, repoHome); assertFileEquals(dump, "[id]", blobsAdded); assertFileEquals(dump, "[ref]", Sets.union(blobsAdded, Sets.newHashSet(deletedBlobId))); assertFileEquals(dump, "[consistency]", Sets.newHashSet(deletedBlobId)); }
@Test public void testReference() throws DataStoreException, IOException { String reference = "testReference"; String blobId = "test"; DataIdentifier testDI = new DataIdentifier(blobId); DataRecord testDR = new ByteArrayDataRecord("foo".getBytes(), testDI, reference); DataStore mockedDS = mock(DataStore.class); when(mockedDS.getRecordFromReference(reference)).thenReturn(testDR); when(mockedDS.getRecord(testDI)).thenReturn(testDR); when(mockedDS.getRecordIfStored(testDI)).thenReturn(testDR); DataStoreBlobStore ds = new DataStoreBlobStore(mockedDS); assertEquals(reference,ds.getReference(blobId)); assertEquals(blobId, BlobId.of(ds.getBlobId(reference)).blobId); assertEquals(BlobId.of(testDR).encodedValue(),ds.getBlobId(reference)); String inMemBlobId = InMemoryDataRecord.getInstance("foo".getBytes()) .getIdentifier().toString(); //For in memory record the reference should be null assertNull(ds.getReference(inMemBlobId)); }
@Test public void testDataStoreBlob() throws Exception { FileDataStore fds = createFileDataStore(); DataStoreBlobStore dbs = new DataStoreBlobStore(fds); nodeStore = getNodeStore(dbs); //Test for Blob which get inlined Blob b1 = testCreateAndRead(createBlob(fds.getMinRecordLength()-2)); assertTrue(b1 instanceof SegmentBlob); assertNull(((SegmentBlob) b1).getBlobId()); //Test for Blob which need to be pushed to BlobStore byte[] data2 = new byte[Segment.MEDIUM_LIMIT + 1]; new Random().nextBytes(data2); Blob b2 = testCreateAndRead(nodeStore.createBlob(new ByteArrayInputStream(data2))); assertTrue(b2 instanceof SegmentBlob); assertNotNull(b2.getReference()); assertEquals(b2.getContentIdentity(), ((SegmentBlob) b2).getBlobId()); InputStream is = dbs.getInputStream(((SegmentBlob) b2).getBlobId()); assertNotNull(IOUtils.contentEquals(new ByteArrayInputStream(data2), is)); is.close(); }
@Test public void testGetAllRecordsWithMeta() throws Exception { File rootFolder = folder.newFolder(); dataStore = getBlobStore(rootFolder); int number = 2; Set<String> added = newHashSet(); for (int i = 0; i < number; i++) { String rec = dataStore.addRecord(randomStream(i, 16516)) .getIdentifier().toString(); added.add(rec); } if (dataStore.getType() == SharedDataStore.Type.SHARED) { dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "1meta"); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "2meta"); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "ameta1"); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), "bmeta2"); } for (int i = 0; i < number; i++) { String rec = dataStore.addRecord(randomStream(100+i, 16516)) .getIdentifier().toString(); added.add(rec); } Set<String> retrieved = newHashSet(Iterables.transform(newHashSet(dataStore.getAllRecords()), new Function<DataRecord, String>() { @Nullable @Override public String apply(@Nullable DataRecord input) { return input.getIdentifier().toString(); } })); assertEquals(added, retrieved); }
@Test public void testGetAllChunkIds() throws Exception { File rootFolder = folder.newFolder(); dataStore = getBlobStore(rootFolder); int number = 10; Set<String> added = newHashSet(); for (int i = 0; i < number; i++) { String rec = dataStore.writeBlob(randomStream(i, 16516)); added.add(rec); } Set<String> retrieved = newHashSet(dataStore.getAllChunkIds(0)); assertEquals(added, retrieved); }
@Override public boolean deleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception { return (chunkIds.size() == countDeleteChunks(chunkIds, maxLastModifiedTime)); }
dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); DataRecord repo1 = dataStore.getMetadataRecord(SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); DataRecord repo2 = dataStore.getMetadataRecord(SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1)); DataRecord markerRec1 = dataStore.getMetadataRecord(SharedStoreRecordType.MARKED_START_MARKER.getNameFromId(repoId1)); assertEquals( SharedStoreRecordType.MARKED_START_MARKER.getIdFromName(markerRec1.getIdentifier().toString()), dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REFERENCES.getNameFromId(repoId1)); DataRecord rec1 = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId1)); long lastModifiedRec1 = rec1.getLastModified(); TimeUnit.MILLISECONDS.sleep(25); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REFERENCES.getNameFromId(repoId2)); DataRecord rec2 = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId2)); long lastModifiedRec2 = rec2.getLastModified(); .refsNotAvailableFromRepos(dataStore.getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType()), dataStore.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType())).isEmpty()); assertEquals( SharedDataStoreUtils.getEarliestRecord( dataStore.getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType())).getLastModified(),
@Override protected void setupCollector(BlobStatsCollector statsCollector) { if (store instanceof DataStoreBlobStore){ ((DataStoreBlobStore) store).setBlobStatsCollector(statsCollector); } }
@Override public int readBlob(String encodedBlobId, long pos, byte[] buff, int off, int length) throws IOException { //This is inefficient as repeated calls for same blobId would involve opening new Stream //instead clients should directly access the stream from DataRecord by special casing for //BlobStore which implements DataStore InputStream stream = getInputStream(encodedBlobId); boolean threw = true; try { ByteStreams.skipFully(stream, pos); int readCount = stream.read(buff, off, length); threw = false; return readCount; } finally { Closeables.close(stream, threw); } }
@Test public void testAddMetadata() throws Exception { File rootFolder = folder.newFolder(); dataStore = getBlobStore(rootFolder); String repoId = UUID.randomUUID().toString(); Set<String> refs = Sets.newHashSet("1_1", "1_2"); File f = folder.newFile(); FileIOUtils.writeStrings(refs.iterator(), f, false); dataStore.addMetadataRecord(new FileInputStream(f), SharedStoreRecordType.REFERENCES.getNameFromId(repoId)); assertTrue(dataStore.metadataRecordExists(SharedStoreRecordType.REFERENCES.getNameFromId(repoId))); DataRecord rec = dataStore.getMetadataRecord(SharedStoreRecordType.REFERENCES.getNameFromId(repoId)); Set<String> refsReturned = FileIOUtils.readStringsAsSet(rec.getStream(), false); assertEquals(refs, refsReturned); assertEquals( SharedStoreRecordType.REFERENCES.getIdFromName(rec.getIdentifier().toString()), repoId); dataStore.deleteAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType()); }
for (int i = 0; i < numBlobs; i++) { Blob b = store.createBlob(randomStream(i, 18342)); Iterator<String> idIter = blobStore.resolveChunks(b.getContentIdentity()); while (idIter.hasNext()) { String chunk = idIter.next(); long count = blobStore.countDeleteChunks(ImmutableList.of(id), 0); assertEquals(1, count);
@Before public void setup() throws Exception { setupDataStore = blobFixture.init(temporaryFolder); store = storeFixture.init(setupDataStore, temporaryFolder.newFolder()); additionalParams = "--ds-read-write"; String repoId = ClusterRepositoryInfo.getOrCreateId(store); setupDataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), REPOSITORY.getNameFromId(repoId)); }
String repoId1 = UUID.randomUUID().toString(); data.repoIds.add(repoId1); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId1)); dataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(repoId2)); dataStore.addMetadataRecord(new FileInputStream(f), getName(extended, repoId1, suffix1)); assertTrue(dataStore.metadataRecordExists(getName(extended, repoId1, suffix1))); DataRecord rec = dataStore.getMetadataRecord(getName(extended, repoId1, suffix1)); assertEquals(refs, FileIOUtils.readStringsAsSet(rec.getStream(), false)); String suffix2 = String.valueOf(System.currentTimeMillis()); data.suffixes.add(suffix2); dataStore.addMetadataRecord(f2, getName(true, repoId1, suffix2)); dataStore.metadataRecordExists(getName(extended, repoId1, suffix2))); String suffix3 = String.valueOf(System.currentTimeMillis()); data.suffixes.add(suffix3); dataStore.addMetadataRecord(f3, getName(extended, repoId2, suffix3)); dataStore.metadataRecordExists(getName(extended, repoId2, suffix3)));