private void createHeader(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() == null, "handle already has a header."); // Create a new Header SegmentChunk. String headerName = StreamSegmentNameUtils.getHeaderSegmentName(handle.getSegmentName()); this.baseStorage.create(headerName); val headerHandle = this.baseStorage.openWrite(headerName); // Create a new Handle and serialize it, after which update the original handle. val newHandle = new RollingSegmentHandle(headerHandle, handle.getRollingPolicy(), handle.chunks()); serializeHandle(newHandle); handle.refresh(newHandle); }
Preconditions.checkState(source.isSealed(), "Cannot concat segment '%s' into '%s' because it is not sealed.", sourceSegment, target.getSegmentName()); if (source.length() == 0) { Preconditions.checkState(source.chunks().stream().allMatch(SegmentChunk::exists), "Cannot use Segment '%s' as concat source because it is truncated.", source.getSegmentName()); SegmentChunk lastTarget = target.lastChunk(); if (lastTarget == null || lastTarget.isSealed()) { SegmentChunk lastSource = source.lastChunk(); this.baseStorage.concat(target.getActiveChunkHandle(), target.lastChunk().getLength(), lastSource.getName()); target.lastChunk().increaseLength(lastSource.getLength()); if (source.getHeaderHandle() != null) { try { this.baseStorage.delete(source.getHeaderHandle()); } catch (StreamSegmentNotExistsException ex) { log.warn("Attempted to delete concat source Header '{}' but it doesn't exist.", source.getHeaderHandle().getSegmentName(), ex); if (target.getHeaderHandle() == null) { List<SegmentChunk> newSegmentChunks = rebase(source.chunks(), target.length()); sealActiveChunk(target); serializeBeginConcat(target, source); this.baseStorage.concat(target.getHeaderHandle(), target.getHeaderLength(), source.getHeaderHandle().getSegmentName()); target.increaseHeaderLength(source.getHeaderLength()); target.addChunks(newSegmentChunks);
private void updateHandle(RollingSegmentHandle handle, byte[] data) throws StreamSegmentException { try { this.baseStorage.write(handle.getHeaderHandle(), handle.getHeaderLength(), new ByteArrayInputStream(data), data.length); handle.increaseHeaderLength(data.length); log.debug("Header for '{}' updated with {} bytes for a length of {}.", handle.getSegmentName(), data.length, handle.getHeaderLength()); } catch (BadOffsetException ex) { // If we get BadOffsetException when writing the Handle, it means it was modified externally. throw new StorageNotPrimaryException(handle.getSegmentName(), ex); } }
/** * Updates the contents of this handle with information from the given one. * * @param source The RollingSegmentHandle to update from. */ synchronized void refresh(RollingSegmentHandle source) { Preconditions.checkArgument(source.getSegmentName().equals(this.getSegmentName()), "SegmentName mismatch."); if (this.readOnly == source.readOnly) { // Update the header handle, but only if both this handle and the source one have the same read-only flag. // Otherwise we risk attaching a read-only header handle to a read-write handle or vice-versa. this.headerHandle = source.headerHandle; } this.segmentChunks = new ArrayList<>(source.chunks()); setHeaderLength(source.getHeaderLength()); if (source.isSealed()) { markSealed(); } if (source.isDeleted()) { markDeleted(); } }
private void ensureOffset(RollingSegmentHandle handle, long offset) throws StreamSegmentException { if (offset != handle.length()) { // Force-refresh the handle to make sure it is still in sync with reality. Make sure we open a read handle // so that we don't force any sort of fencing during this process. val refreshedHandle = openHandle(handle.getSegmentName(), true); handle.refresh(refreshedHandle); log.debug("Handle refreshed: {}.", handle); if (offset != handle.length()) { // Still in disagreement; throw exception. throw new BadOffsetException(handle.getSegmentName(), handle.length(), offset); } } }
public void testMainFeatures() { val headerHandle = new TestHandle(HEADER_NAME, true); val h = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, new ArrayList<>()); Assert.assertNull("Unexpected lastChunk for empty handle.", h.lastChunk()); Assert.assertEquals("Unexpected contents in chunks() for empty handle.", 0, h.chunks().size()); Assert.assertEquals("Unexpected value for length() for empty handle.", 0, h.length()); Assert.assertEquals("Unexpected value for getHeaderLength() for empty handle.", 0, h.getHeaderLength()); Assert.assertEquals("Unexpected value for getHeaderHandle().", headerHandle, h.getHeaderHandle()); Assert.assertEquals("Unexpected segment name.", SEGMENT_NAME, h.getSegmentName()); Assert.assertEquals("Unexpected rolling policy.", DEFAULT_ROLLING_POLICY, h.getRollingPolicy()); Assert.assertTrue("Unexpected value for isReadOnly.", h.isReadOnly()); Assert.assertFalse("Unexpected value for isSealed.", h.isSealed()); Assert.assertFalse("Unexpected value for isDeleted.", h.isDeleted()); () -> h.setActiveChunkHandle(new TestHandle("foo", false)), ex -> ex instanceof IllegalStateException); h.addChunk(new SegmentChunk(chunkName, 0L), new TestHandle(chunkName, false)); AssertExtensions.assertThrows( "setActiveChunkHandle accepted a handle that does not match the last SegmentChunk's name.", () -> h.setActiveChunkHandle(new TestHandle("foo", false)), ex -> ex instanceof IllegalArgumentException); () -> h.setActiveChunkHandle(new TestHandle(chunkName, true)), ex -> ex instanceof IllegalArgumentException); h.setActiveChunkHandle(activeHandle); Assert.assertEquals("Unexpected value from getActiveChunkHandle.", activeHandle, h.getActiveChunkHandle());
/** * Tests the ability of the Handle to refresh based on information from another similar handle. */ @Test public void testRefresh() { val headerHandle = new TestHandle(HEADER_NAME, true); val target = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, Collections.singletonList(new SegmentChunk("s1", 0L))); val source = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, Arrays.asList( new SegmentChunk("s1", 0L), new SegmentChunk("s2", 100L))); source.chunks().get(0).setLength(100); source.markSealed(); source.setHeaderLength(1000); source.setActiveChunkHandle(new TestHandle(source.lastChunk().getName(), false)); target.refresh(source); Assert.assertEquals("Unexpected getHeaderLength()", source.getHeaderLength(), target.getHeaderLength()); AssertExtensions.assertListEquals("Unexpected chunks()", source.chunks(), target.chunks(), Object::equals); Assert.assertTrue("Unexpected isSealed.", target.isSealed()); Assert.assertNull("Not expecting any ActiveSegmentHandle to be copied.", target.getActiveChunkHandle()); }
private void rollover(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() != null, "Cannot rollover a Segment with no header."); Preconditions.checkArgument(!handle.isReadOnly(), "Cannot rollover using a read-only handle."); Preconditions.checkArgument(!handle.isSealed(), "Cannot rollover a Sealed Segment."); log.debug("Rolling over '{}'.", handle); sealActiveChunk(handle); try { createChunk(handle); } catch (StreamSegmentExistsException ex) { // It may be possible that a concurrent rollover request using a different handle (either from this instance // or another) has already created the new chunk. Refresh the handle and try again. This is usually the case // with concurrent Storage instances trying to modify the same segment at the same time. int chunkCount = handle.chunks().size(); handle.refresh(openHandle(handle.getSegmentName(), false)); if (chunkCount == handle.chunks().size()) { // Nothing changed; re-throw the exception. throw ex; } else { // We've just refreshed the handle and picked up the latest chunk. Move on. log.warn("Aborted rollover due to concurrent rollover detected ('{}').", handle); } } }
public void testAddChunk() { val headerHandle = new TestHandle(HEADER_NAME, true); val h = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, new ArrayList<>()); () -> h.addChunk(new SegmentChunk("s", 0L), null), ex -> ex instanceof NullPointerException); AssertExtensions.assertThrows( "addChunk allowed adding a read-only ActiveSegmentHandle.", () -> h.addChunk(new SegmentChunk("s", 0L), new TestHandle("s", true)), ex -> ex instanceof IllegalArgumentException); AssertExtensions.assertThrows( "addChunk allowed adding an ActiveSegmentHandle with different name..", () -> h.addChunk(new SegmentChunk("s", 0L), new TestHandle("s2", false)), ex -> ex instanceof IllegalArgumentException); Assert.assertEquals("Not expecting any SegmentChunks to be added.", 0, h.chunks().size()); Assert.assertNull("Not expecting the Active SegmentChunk handle to be set.", h.getActiveChunkHandle()); h.addChunk(chunk, new TestHandle("s1", false)); chunk.setLength(123L); Assert.assertEquals("Unexpected value for length() after adding one SegmentChunk.", chunk.getStartOffset() + chunk.getLength(), h.length()); AssertExtensions.assertListEquals("Unexpected contents for chunks().", Collections.singletonList(chunk), h.chunks(), Object::equals); Assert.assertEquals("Unexpected lastChunk.", chunk, h.lastChunk()); () -> h.addChunk(new SegmentChunk("s2", chunk.getLastOffset() + 1), new TestHandle("s2", false)), ex -> ex instanceof IllegalArgumentException); val chunk2 = new SegmentChunk("s2", chunk.getLastOffset());
if (h.isReadOnly() && !h.isSealed() && offset + length > h.length()) { h.refresh(newHandle); log.debug("Handle refreshed: {}.", h); Preconditions.checkArgument(offset + length <= h.length(), "Offset %s + length %s is beyond the last offset %s of the segment.", offset, length, h.length()); val chunks = h.chunks(); int currentIndex = CollectionHelpers.binarySearch(chunks, s -> offset < s.getStartOffset() ? -1 : (offset >= s.getLastOffset() ? 1 : 0)); assert currentIndex >= 0 : "unable to locate first SegmentChunk index."; h.refresh(newHandle); if (h.isDeleted()) { log.debug("Segment '{}' has been deleted. Cannot read anymore.", h); throw new StreamSegmentNotExistsException(handle.getSegmentName(), ex);
os.write(serialize(source)); for (int i = 0; i < concatCount; i++) { os.write(HandleSerializer.serializeConcat(chunkCount, source.length())); if (i % failConcatEvery != 0) { source.addChunks(concatHandle.chunks().stream() .map(s -> s.withNewOffset(s.getStartOffset() + source.length())).collect(Collectors.toList())); val chunk = new SegmentChunk(StreamSegmentNameUtils.getSegmentChunkName(source.getSegmentName(), source.length()), source.length()); chunk.setLength(i + 1); source.addChunks(Collections.singletonList(chunk)); os.write(HandleSerializer.serializeChunk(chunk)); val newHandle = HandleSerializer.deserialize(serialization, source.getHeaderHandle()); assertHandleEquals(source, newHandle, source.getHeaderHandle()); Assert.assertEquals("getHeaderLength", serialization.length, newHandle.getHeaderLength());
long traceId = LoggerHelpers.traceEnter(log, "delete", handle); SegmentHandle headerHandle = h.getHeaderHandle(); if (headerHandle == null) { val subHandle = this.baseStorage.openWrite(h.lastChunk().getName()); try { this.baseStorage.delete(subHandle); h.lastChunk().markInexistent(); h.markDeleted(); } catch (StreamSegmentNotExistsException ex) { h.lastChunk().markInexistent(); h.markDeleted(); throw ex; if (!h.isSealed()) { val writeHandle = h.isReadOnly() ? (RollingSegmentHandle) openWrite(handle.getSegmentName()) : h; seal(writeHandle); try { this.baseStorage.delete(headerHandle); h.markDeleted(); } catch (StreamSegmentNotExistsException ex) { h.markDeleted(); throw ex;
@Override public void truncate(SegmentHandle handle, long truncationOffset) throws StreamSegmentException { // Delete all SegmentChunks which are entirely before the truncation offset. RollingSegmentHandle h = asReadableHandle(handle); ensureNotDeleted(h); // The only acceptable case where we allow a read-only handle is if the Segment is sealed, since openWrite() will // only return a read-only handle in that case. Preconditions.checkArgument(h.isSealed() || !h.isReadOnly(), "Can only truncate with a read-only handle if the Segment is Sealed."); if (h.getHeaderHandle() == null) { // No header means the Segment is made up of a single SegmentChunk. We can't do anything. return; } long traceId = LoggerHelpers.traceEnter(log, "truncate", h, truncationOffset); Preconditions.checkArgument(truncationOffset >= 0 && truncationOffset <= h.length(), "truncationOffset must be non-negative and at most the length of the Segment."); val last = h.lastChunk(); if (last != null && canTruncate(last, truncationOffset) && !h.isSealed()) { // If we were asked to truncate the entire (non-sealed) Segment, then rollover at this point so we can delete // all existing data. rollover(h); // We are free to delete all chunks. deleteChunks(h, s -> canTruncate(s, truncationOffset)); } else { // Either we were asked not to truncate the whole segment, or we were, and the Segment is sealed. If the latter, // then the Header is also sealed, we could not have done a quick rollover; as such we have no option but to // preserve the last chunk so that we can recalculate the length of the Segment if we need it again. deleteChunks(h, s -> canTruncate(s, truncationOffset) && s.getLastOffset() < h.length()); } LoggerHelpers.traceLeave(log, "truncate", traceId, h, truncationOffset); }
private void assertHandleEquals(RollingSegmentHandle expected, RollingSegmentHandle actual, SegmentHandle headerHandle) { Assert.assertEquals("getSegmentName", expected.getSegmentName(), actual.getSegmentName()); AssertExtensions.assertListEquals("chunks", expected.chunks(), actual.chunks(), this::chunkEquals); Assert.assertEquals("getRollingPolicy", expected.getRollingPolicy().getMaxLength(), actual.getRollingPolicy().getMaxLength()); Assert.assertEquals("getHeaderHandle", headerHandle, actual.getHeaderHandle()); Assert.assertEquals("isReadOnly", headerHandle.isReadOnly(), expected.isReadOnly()); }
private void sealActiveChunk(RollingSegmentHandle handle) throws StreamSegmentException { SegmentHandle activeChunk = handle.getActiveChunkHandle(); SegmentChunk last = handle.lastChunk(); if (activeChunk != null && !last.isSealed()) { this.baseStorage.seal(activeChunk); handle.setActiveChunkHandle(null); last.markSealed(); log.debug("Sealed active SegmentChunk '{}' for '{}'.", activeChunk.getSegmentName(), handle.getSegmentName()); } }
public void testAddChunks() { val headerHandle = new TestHandle(HEADER_NAME, true); val h = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, new ArrayList<>()); AssertExtensions.assertThrows( "addChunks allowed an incontiguous list of SegmentChunks to be added.", () -> h.addChunks(firstBadList), ex -> ex instanceof IllegalArgumentException); Assert.assertEquals("Not expecting any SegmentChunks to be added.", 0, h.chunks().size()); Assert.assertEquals("Unexpected length().", 0, h.length()); validList.get(0).setLength(10); validList.get(1).setLength(5); h.addChunks(validList); AssertExtensions.assertListEquals("Unexpected list of SegmentChunks.", validList, h.chunks(), Object::equals); Assert.assertEquals("Unexpected length.", 15, h.length()); new SegmentChunk("s3", h.length() - 1), new SegmentChunk("s4", h.length() + 1)); secondBadList.get(0).setLength(2); AssertExtensions.assertThrows( "addChunks allowed an incontiguous list of SegmentChunks to be added.", () -> h.addChunks(secondBadList), ex -> ex instanceof IllegalArgumentException);
handle = new RollingSegmentHandle(segmentHandle); for (SegmentChunk s : handle.chunks()) { if (last != null) { last.setLength(s.getStartOffset() - last.getStartOffset()); if (si.isSealed()) { last.markSealed(); if (handle.getHeaderHandle() == null) { handle.markSealed();
private boolean shouldConcatNatively(RollingSegmentHandle source, RollingSegmentHandle target) { if (source.getHeaderHandle() == null) { // Source does not have a Header, hence we cannot do Header concat. return true; } SegmentChunk lastSource = source.lastChunk(); SegmentChunk lastTarget = target.lastChunk(); return lastSource != null && lastSource.getStartOffset() == 0 && lastTarget != null && !lastTarget.isSealed() && lastTarget.getLength() + lastSource.getLength() <= target.getRollingPolicy().getMaxLength(); }
private void serializeHandle(RollingSegmentHandle handle) throws StreamSegmentException { ByteArraySegment handleData = HandleSerializer.serialize(handle); try { this.baseStorage.write(handle.getHeaderHandle(), 0, handleData.getReader(), handleData.getLength()); handle.setHeaderLength(handleData.getLength()); log.debug("Header for '{}' fully serialized to '{}'.", handle.getSegmentName(), handle.getHeaderHandle().getSegmentName()); } catch (BadOffsetException ex) { // If we get BadOffsetException when writing the Handle, it means it was modified externally. throw new StorageNotPrimaryException(handle.getSegmentName(), ex); } }
@Override public SegmentProperties getStreamSegmentInfo(String segmentName) throws StreamSegmentException { val handle = (RollingSegmentHandle) openRead(segmentName); return StreamSegmentInformation .builder() .name(handle.getSegmentName()) .sealed(handle.isSealed()) .length(handle.length()) .build(); }