@Override public Storage createStorageAdapter() { return new AsyncStorageWrapper(new RollingStorage(this.baseStorage), this.executor); }
@Override public SegmentHandle create(String streamSegmentName) throws StreamSegmentException { return create(streamSegmentName, this.defaultRollingPolicy); }
@Override @SneakyThrows(StreamSegmentException.class) public boolean exists(String segmentName) { try { // Try to open-read the segment, this checks both the header file and the existence of the last SegmentChunk. openRead(segmentName); return true; } catch (StreamSegmentNotExistsException ex) { return false; } }
/** * Tests the ability to concat using the header file for those cases when native concat cannot be used because the * source Segment has a single SegmentChunk, but it's too large to fit into the Target's active SegmentChunk. */ @Test public void testConcatHeaderSingleFile() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final int bigSourceLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() - initialTargetLength + 1; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Target Segment and a Source Segment and write some data to them. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, 1, bigSourceLength, bigSourceLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 2, initialTargetLength + bigSourceLength); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); () -> s.create(segmentName), ex -> ex instanceof StreamSegmentExistsException); Assert.assertTrue("Non-Header Segment does not exist after failed create() attempt.", baseStorage.exists(segmentName)); Assert.assertTrue("Unexpected result from exists() when called on a non-header Segment.", s.exists(segmentName)); val writeHandle = s.openWrite(segmentName); val os = new ByteArrayOutputStream(); populate(s, writeHandle, os); s.seal(writeHandle); byte[] writtenData = os.toByteArray(); Assert.assertFalse("A header was left behind (after write).", val rollingInfo = s.getStreamSegmentInfo(segmentName); Assert.assertTrue("Segment not sealed.", baseInfo.isSealed()); Assert.assertEquals("Unexpected Segment length.", writtenData.length, baseInfo.getLength()); val readHandle = s.openRead(segmentName); checkWrittenData(writtenData, readHandle, s); s.truncate(writeHandle, truncateOffset); val nonHeaderHandle = s.openWrite(nonHeaderName); s.concat(nonHeaderHandle, 0, segmentName);
@Override public void concat(SegmentHandle targetHandle, long targetOffset, String sourceSegment) throws StreamSegmentException { val target = asWritableHandle(targetHandle); ensureOffset(target, targetOffset); ensureNotDeleted(target); ensureNotSealed(target); long traceId = LoggerHelpers.traceEnter(log, "concat", target, targetOffset, sourceSegment); RollingSegmentHandle source = (RollingSegmentHandle) openWrite(sourceSegment); Preconditions.checkState(source.isSealed(), "Cannot concat segment '%s' into '%s' because it is not sealed.", sourceSegment, target.getSegmentName()); delete(source); return; refreshChunkExistence(source); Preconditions.checkState(source.chunks().stream().allMatch(SegmentChunk::exists), "Cannot use Segment '%s' as concat source because it is truncated.", source.getSegmentName()); if (shouldConcatNatively(source, target)) { rollover(target); createHeader(target); List<SegmentChunk> newSegmentChunks = rebase(source.chunks(), target.length()); sealActiveChunk(target); serializeBeginConcat(target, source); this.baseStorage.concat(target.getHeaderHandle(), target.getHeaderLength(), source.getHeaderHandle().getSegmentName()); target.increaseHeaderLength(source.getHeaderLength());
val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME); val writeHandle = s.openWrite(SEGMENT_NAME); val readHandle = s.openRead(SEGMENT_NAME); // Open now, before writing, so we force a refresh. val writeStream = new ByteArrayOutputStream(); populate(s, writeHandle, writeStream); Assert.assertEquals("Unexpected segment length.", writtenData.length, s.getStreamSegmentInfo(SEGMENT_NAME).getLength()); int checkedLength = 0; while (checkedLength < writtenData.length) {
val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME); val writeHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); populate(s, writeHandle, null); AssertExtensions.assertThrows( "delete() did not propagate proper exception on failure.", () -> s.delete(writeHandle), ex -> ex instanceof IntentionalException); Assert.assertTrue("Not expecting segment to be deleted yet.", s.exists(SEGMENT_NAME)); Assert.assertFalse("Expected first SegmentChunk to be marked as deleted.", writeHandle.chunks().get(failAtIndex - 1).exists()); Assert.assertTrue("Expected failed-to-delete SegmentChunk to not be marked as deleted.", writeHandle.chunks().get(failAtIndex).exists()); s.delete(writeHandle); Assert.assertFalse("Expecting the segment to be deleted.", s.exists(SEGMENT_NAME)); Assert.assertTrue("Expected the handle to be marked as deleted.", writeHandle.isDeleted()); Assert.assertFalse("Expected all SegmentChunks to be marked as deleted.", writeHandle.chunks().stream().anyMatch(SegmentChunk::exists));
/** * Tests the ability to truncate Sealed Segments. */ @Test public void testTruncateSealed() throws Exception { // Write small and large writes, alternatively. @Cleanup val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Segment, write some data, then seal it. s.create(SEGMENT_NAME); val appendHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, appendHandle, writeStream); s.seal(appendHandle); byte[] writtenData = writeStream.toByteArray(); val truncateHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); Assert.assertTrue("Handle not read-only after sealing.", truncateHandle.isReadOnly()); Assert.assertTrue("Handle not sealed after sealing.", truncateHandle.isSealed()); // Test that truncate works in this scenario. testProgressiveTruncate(truncateHandle, truncateHandle, writtenData, s, baseStorage); }
/** * Tests the case when Create was interrupted after it created the Header file but before populating it. */ @Test public void testCreateRecovery() throws Exception { @Cleanup val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create an empty header file. This simulates a create() operation that failed mid-way. baseStorage.create(StreamSegmentNameUtils.getHeaderSegmentName(SEGMENT_NAME)); Assert.assertFalse("Not expecting Segment to exist.", s.exists(SEGMENT_NAME)); AssertExtensions.assertThrows( "Not expecting Segment to exist (getStreamSegmentInfo).", () -> s.getStreamSegmentInfo(SEGMENT_NAME), ex -> ex instanceof StreamSegmentNotExistsException); AssertExtensions.assertThrows( "Not expecting Segment to exist (openHandle).", () -> s.openRead(SEGMENT_NAME), ex -> ex instanceof StreamSegmentNotExistsException); // Retry the operation and verify everything is in place. s.create(SEGMENT_NAME); val si = s.getStreamSegmentInfo(SEGMENT_NAME); Assert.assertEquals("Expected the Segment to have been created.", 0, si.getLength()); }
/** * Tests the ability to auto-refresh a Write Handle upon offset disagreement. */ @Test public void testRefreshHandleBadOffset() throws Exception { // Write small and large writes, alternatively. @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME); val h1 = s.openWrite(SEGMENT_NAME); val h2 = s.openWrite(SEGMENT_NAME); // Open now, before writing, so we force a refresh. byte[] data = "data".getBytes(); s.write(h1, 0, new ByteArrayInputStream(data), data.length); s.write(h2, data.length, new ByteArrayInputStream(data), data.length); // Check that no file has exceeded its maximum length. byte[] expectedData = new byte[data.length * 2]; System.arraycopy(data, 0, expectedData, 0, data.length); System.arraycopy(data, 0, expectedData, data.length, data.length); checkWrittenData(expectedData, h2, s); }
@Override public void delete(SegmentHandle handle) throws StreamSegmentException { val h = asReadableHandle(handle); long traceId = LoggerHelpers.traceEnter(log, "delete", handle); val writeHandle = h.isReadOnly() ? (RollingSegmentHandle) openWrite(handle.getSegmentName()) : h; seal(writeHandle); deleteChunks(h, s -> true); try { this.baseStorage.delete(headerHandle);
@Override public int read(SegmentHandle handle, long offset, byte[] buffer, int bufferOffset, int length) throws StreamSegmentException { val h = asReadableHandle(handle); long traceId = LoggerHelpers.traceEnter(log, "read", handle, offset, length); ensureNotDeleted(h); Exceptions.checkArrayRange(bufferOffset, length, buffer.length, "bufferOffset", "length"); val newHandle = (RollingSegmentHandle) openRead(handle.getSegmentName()); h.refresh(newHandle); log.debug("Handle refreshed: {}.", h); checkTruncatedSegment(null, h, current); if (current.getLength() == 0) { checkTruncatedSegment(ex, h, current); val newHandle = (RollingSegmentHandle) openRead(handle.getSegmentName()); h.refresh(newHandle); if (h.isDeleted()) {
private void checkConcatResult(RollingStorage s, RollingSegmentHandle targetHandle, String sourceSegmentName, int expectedChunkCount, int expectedLength) throws Exception { Assert.assertFalse("Expecting the source segment to not exist anymore.", s.exists(sourceSegmentName)); Assert.assertEquals("Unexpected number of SegmentChunks in target.", expectedChunkCount, targetHandle.chunks().size()); Assert.assertEquals("Unexpected target length.", expectedLength, targetHandle.length()); // Reload the handle and verify nothing strange happened in Storage. val targetHandle2 = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); Assert.assertEquals("Unexpected number of SegmentChunks in reloaded target handle.", expectedChunkCount, targetHandle2.chunks().size()); Assert.assertEquals("Unexpected reloaded target length.", targetHandle.length(), targetHandle2.length()); }
/** * Tests the ability to concat using the header file for those cases when native concat cannot be used because the * source Segment has multiple SegmentChunks. */ @Test public void testConcatHeaderMultiFile() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Target Segment and a Source Segment and write some data to them. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, APPENDS_PER_SEGMENT, initialTargetLength, initialTargetLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 1 + sourceHandle.chunks().size(), initialTargetLength + (int) sourceHandle.length()); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
/** * Tests the ability to use native concat for those cases when it's appropriate. */ @Test public void testConcatNatively() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final int initialSourceLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() - initialTargetLength; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a target Segment and write a little data to it. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); // Create a source Segment and write a little data to it, making sure it is small enough to fit into the target // when we need to concat. s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, 1, initialSourceLength, initialSourceLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 1, initialTargetLength + initialSourceLength); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
@Override public Storage createStorageAdapter() { HDFSStorage s = new HDFSStorage(this.config); return new AsyncStorageWrapper(new RollingStorage(s), this.executor); } }
@Override public SegmentProperties getStreamSegmentInfo(String segmentName) throws StreamSegmentException { val handle = (RollingSegmentHandle) openRead(segmentName); return StreamSegmentInformation .builder() .name(handle.getSegmentName()) .sealed(handle.isSealed()) .length(handle.length()) .build(); }
val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME); val writeHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val readHandle = s.openRead(SEGMENT_NAME); // Open now, before writing, so we force a refresh. val writeStream = new ByteArrayOutputStream(); populate(s, writeHandle, writeStream); s.create(targetSegmentName); val targetSegmentHandle = s.openWrite(targetSegmentName); s.seal(writeHandle); AssertExtensions.assertThrows( "concat() allowed using a truncated segment as a source.", () -> s.concat(targetSegmentHandle, 0, SEGMENT_NAME), ex -> ex instanceof IllegalStateException);
@Override public Storage createStorageAdapter() { FileSystemStorage s = new FileSystemStorage(this.config); return new AsyncStorageWrapper(new RollingStorage(s), this.executor); } }