private void updateHandle(RollingSegmentHandle handle, byte[] data) throws StreamSegmentException { try { this.baseStorage.write(handle.getHeaderHandle(), handle.getHeaderLength(), new ByteArrayInputStream(data), data.length); handle.increaseHeaderLength(data.length); log.debug("Header for '{}' updated with {} bytes for a length of {}.", handle.getSegmentName(), data.length, handle.getHeaderLength()); } catch (BadOffsetException ex) { // If we get BadOffsetException when writing the Handle, it means it was modified externally. throw new StorageNotPrimaryException(handle.getSegmentName(), ex); } }
private void rollover(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() != null, "Cannot rollover a Segment with no header."); Preconditions.checkArgument(!handle.isReadOnly(), "Cannot rollover using a read-only handle."); Preconditions.checkArgument(!handle.isSealed(), "Cannot rollover a Sealed Segment."); log.debug("Rolling over '{}'.", handle); sealActiveChunk(handle); try { createChunk(handle); } catch (StreamSegmentExistsException ex) { // It may be possible that a concurrent rollover request using a different handle (either from this instance // or another) has already created the new chunk. Refresh the handle and try again. This is usually the case // with concurrent Storage instances trying to modify the same segment at the same time. int chunkCount = handle.chunks().size(); handle.refresh(openHandle(handle.getSegmentName(), false)); if (chunkCount == handle.chunks().size()) { // Nothing changed; re-throw the exception. throw ex; } else { // We've just refreshed the handle and picked up the latest chunk. Move on. log.warn("Aborted rollover due to concurrent rollover detected ('{}').", handle); } } }
private void serializeHandle(RollingSegmentHandle handle) throws StreamSegmentException { ByteArraySegment handleData = HandleSerializer.serialize(handle); try { this.baseStorage.write(handle.getHeaderHandle(), 0, handleData.getReader(), handleData.getLength()); handle.setHeaderLength(handleData.getLength()); log.debug("Header for '{}' fully serialized to '{}'.", handle.getSegmentName(), handle.getHeaderHandle().getSegmentName()); } catch (BadOffsetException ex) { // If we get BadOffsetException when writing the Handle, it means it was modified externally. throw new StorageNotPrimaryException(handle.getSegmentName(), ex); } }
@Override public void seal(SegmentHandle handle) throws StreamSegmentException { val h = asWritableHandle(handle); ensureNotDeleted(h); long traceId = LoggerHelpers.traceEnter(log, "seal", handle); sealActiveChunk(h); SegmentHandle headerHandle = h.getHeaderHandle(); if (headerHandle != null) { this.baseStorage.seal(headerHandle); } h.markSealed(); log.debug("Sealed Header for '{}'.", h.getSegmentName()); LoggerHelpers.traceLeave(log, "seal", traceId, handle); }
private void createHeader(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() == null, "handle already has a header."); // Create a new Header SegmentChunk. String headerName = StreamSegmentNameUtils.getHeaderSegmentName(handle.getSegmentName()); this.baseStorage.create(headerName); val headerHandle = this.baseStorage.openWrite(headerName); // Create a new Handle and serialize it, after which update the original handle. val newHandle = new RollingSegmentHandle(headerHandle, handle.getRollingPolicy(), handle.chunks()); serializeHandle(newHandle); handle.refresh(newHandle); }
/** * Tests the basic Serialization-Deserialization for a Handle with no concat executed on it. */ @Test public void testNormalSerialization() { final int chunkCount = 1000; val source = newHandle(chunkCount); val serialization = serialize(source); val newHandle = HandleSerializer.deserialize(serialization, source.getHeaderHandle()); assertHandleEquals(source, newHandle, source.getHeaderHandle()); Assert.assertEquals("getHeaderLength", serialization.length, newHandle.getHeaderLength()); }
private boolean shouldConcatNatively(RollingSegmentHandle source, RollingSegmentHandle target) { if (source.getHeaderHandle() == null) { // Source does not have a Header, hence we cannot do Header concat. return true; } SegmentChunk lastSource = source.lastChunk(); SegmentChunk lastTarget = target.lastChunk(); return lastSource != null && lastSource.getStartOffset() == 0 && lastTarget != null && !lastTarget.isSealed() && lastTarget.getLength() + lastSource.getLength() <= target.getRollingPolicy().getMaxLength(); }
this.baseStorage.concat(target.getActiveChunkHandle(), target.lastChunk().getLength(), lastSource.getName()); target.lastChunk().increaseLength(lastSource.getLength()); if (source.getHeaderHandle() != null) { try { this.baseStorage.delete(source.getHeaderHandle()); } catch (StreamSegmentNotExistsException ex) { log.warn("Attempted to delete concat source Header '{}' but it doesn't exist.", source.getHeaderHandle().getSegmentName(), ex); if (target.getHeaderHandle() == null) { sealActiveChunk(target); serializeBeginConcat(target, source); this.baseStorage.concat(target.getHeaderHandle(), target.getHeaderLength(), source.getHeaderHandle().getSegmentName()); target.increaseHeaderLength(source.getHeaderLength()); target.addChunks(newSegmentChunks);
@Override public void truncate(SegmentHandle handle, long truncationOffset) throws StreamSegmentException { // Delete all SegmentChunks which are entirely before the truncation offset. RollingSegmentHandle h = asReadableHandle(handle); ensureNotDeleted(h); // The only acceptable case where we allow a read-only handle is if the Segment is sealed, since openWrite() will // only return a read-only handle in that case. Preconditions.checkArgument(h.isSealed() || !h.isReadOnly(), "Can only truncate with a read-only handle if the Segment is Sealed."); if (h.getHeaderHandle() == null) { // No header means the Segment is made up of a single SegmentChunk. We can't do anything. return; } long traceId = LoggerHelpers.traceEnter(log, "truncate", h, truncationOffset); Preconditions.checkArgument(truncationOffset >= 0 && truncationOffset <= h.length(), "truncationOffset must be non-negative and at most the length of the Segment."); val last = h.lastChunk(); if (last != null && canTruncate(last, truncationOffset) && !h.isSealed()) { // If we were asked to truncate the entire (non-sealed) Segment, then rollover at this point so we can delete // all existing data. rollover(h); // We are free to delete all chunks. deleteChunks(h, s -> canTruncate(s, truncationOffset)); } else { // Either we were asked not to truncate the whole segment, or we were, and the Segment is sealed. If the latter, // then the Header is also sealed, we could not have done a quick rollover; as such we have no option but to // preserve the last chunk so that we can recalculate the length of the Segment if we need it again. deleteChunks(h, s -> canTruncate(s, truncationOffset) && s.getLastOffset() < h.length()); } LoggerHelpers.traceLeave(log, "truncate", traceId, h, truncationOffset); }
val newHandle = HandleSerializer.deserialize(serialization, source.getHeaderHandle()); assertHandleEquals(source, newHandle, source.getHeaderHandle()); Assert.assertEquals("getHeaderLength", serialization.length, newHandle.getHeaderLength());
baseStorage.concatFailure = sn -> sn.equals(sourceHandle.getHeaderHandle().getSegmentName()) ? new IntentionalException() : null; for (int i = 0; i < 4; i++) { AssertExtensions.assertThrows(
baseStorage.deleteFailure = sn -> sn.equals(sourceHandle.getHeaderHandle().getSegmentName()) ? new IntentionalException() : null; AssertExtensions.assertThrows( "Unexpected exception when doing native concat.",
if (si.isSealed()) { last.markSealed(); if (handle.getHeaderHandle() == null) { handle.markSealed();
private void assertHandleEquals(RollingSegmentHandle expected, RollingSegmentHandle actual, SegmentHandle headerHandle) { Assert.assertEquals("getSegmentName", expected.getSegmentName(), actual.getSegmentName()); AssertExtensions.assertListEquals("chunks", expected.chunks(), actual.chunks(), this::chunkEquals); Assert.assertEquals("getRollingPolicy", expected.getRollingPolicy().getMaxLength(), actual.getRollingPolicy().getMaxLength()); Assert.assertEquals("getHeaderHandle", headerHandle, actual.getHeaderHandle()); Assert.assertEquals("isReadOnly", headerHandle.isReadOnly(), expected.isReadOnly()); }
long traceId = LoggerHelpers.traceEnter(log, "delete", handle); SegmentHandle headerHandle = h.getHeaderHandle(); if (headerHandle == null) {
Assert.assertEquals("Unexpected value for length() for empty handle.", 0, h.length()); Assert.assertEquals("Unexpected value for getHeaderLength() for empty handle.", 0, h.getHeaderLength()); Assert.assertEquals("Unexpected value for getHeaderHandle().", headerHandle, h.getHeaderHandle()); Assert.assertEquals("Unexpected segment name.", SEGMENT_NAME, h.getSegmentName()); Assert.assertEquals("Unexpected rolling policy.", DEFAULT_ROLLING_POLICY, h.getRollingPolicy());