private List<SegmentChunk> rebase(List<SegmentChunk> segmentChunks, long newStartOffset) { AtomicLong segmentOffset = new AtomicLong(newStartOffset); return segmentChunks.stream() .map(s -> s.withNewOffset(segmentOffset.getAndAdd(s.getLength()))) .collect(Collectors.toList()); }
/** * Adds multiple SegmentChunks. * * @param segmentChunks The SegmentChunks to add. These SegmentChunks must be in continuity of any existing SegmentChunks. */ synchronized void addChunks(List<SegmentChunk> segmentChunks) { Preconditions.checkState(!this.sealed, "Cannot add SegmentChunks for a Sealed Handle."); long expectedOffset = 0; if (this.segmentChunks.size() > 0) { expectedOffset = this.segmentChunks.get(this.segmentChunks.size() - 1).getLastOffset(); } else if (segmentChunks.size() > 0) { expectedOffset = segmentChunks.get(0).getStartOffset(); } for (SegmentChunk s : segmentChunks) { Preconditions.checkArgument(s.getStartOffset() == expectedOffset, "Invalid SegmentChunk StartOffset. Expected %s, given %s.", expectedOffset, s.getStartOffset()); expectedOffset += s.getLength(); } this.segmentChunks.addAll(segmentChunks); this.activeChunkHandle = null; }
private boolean shouldConcatNatively(RollingSegmentHandle source, RollingSegmentHandle target) { if (source.getHeaderHandle() == null) { // Source does not have a Header, hence we cannot do Header concat. return true; } SegmentChunk lastSource = source.lastChunk(); SegmentChunk lastTarget = target.lastChunk(); return lastSource != null && lastSource.getStartOffset() == 0 && lastTarget != null && !lastTarget.isSealed() && lastTarget.getLength() + lastSource.getLength() <= target.getRollingPolicy().getMaxLength(); }
/** * Creates a new instance of the SegmentChunk class with the same information as this one, but with a new offset. * * @param newOffset The new offset. * @return A new SegmentChunk. */ SegmentChunk withNewOffset(long newOffset) { SegmentChunk ns = new SegmentChunk(this.name, newOffset); ns.setLength(getLength()); if (isSealed()) { ns.markSealed(); } if (!exists()) { ns.markInexistent(); } return ns; }
if (current.getLength() == 0) { int readLength = (int) Math.min(length - bytesRead, current.getLength() - readOffset); assert readOffset >= 0 && readLength >= 0 : "negative readOffset or readLength"; int count = this.baseStorage.read(sh, readOffset, buffer, bufferOffset + bytesRead, readLength); bytesRead += count; if (readOffset + count >= current.getLength()) { currentIndex++;
private RollingSegmentHandle newHandle(String segmentName, int chunkCount) { val chunks = new ArrayList<SegmentChunk>(); long offset = 0; val rnd = new Random(0); for (int i = 0; i < chunkCount; i++) { val chunk = new SegmentChunk(StreamSegmentNameUtils.getSegmentChunkName(segmentName, offset), offset); chunk.setLength(MathHelpers.abs(rnd.nextInt())); if (i < chunkCount - 1) { chunk.markSealed(); } chunks.add(chunk); offset += chunk.getLength(); } return new RollingSegmentHandle(new TestHandle(StreamSegmentNameUtils.getHeaderSegmentName(segmentName), false), TEST_ROLLING_POLICY, chunks); }
@Override public void write(SegmentHandle handle, long offset, InputStream data, int length) throws StreamSegmentException { val h = asWritableHandle(handle); ensureNotDeleted(h); ensureNotSealed(h); ensureOffset(h, offset); long traceId = LoggerHelpers.traceEnter(log, "write", handle, offset, length); // We run this in a loop because we may have to split the write over multiple SegmentChunks in order to avoid exceeding // any SegmentChunk's maximum length. int bytesWritten = 0; while (bytesWritten < length) { if (h.getActiveChunkHandle() == null || h.lastChunk().getLength() >= h.getRollingPolicy().getMaxLength()) { rollover(h); } SegmentChunk last = h.lastChunk(); int writeLength = (int) Math.min(length - bytesWritten, h.getRollingPolicy().getMaxLength() - last.getLength()); assert writeLength > 0 : "non-positive write length"; long chunkOffset = offset + bytesWritten - last.getStartOffset(); this.baseStorage.write(h.getActiveChunkHandle(), chunkOffset, data, writeLength); last.increaseLength(writeLength); bytesWritten += writeLength; } LoggerHelpers.traceLeave(log, "write", traceId, handle, offset, bytesWritten); }
chunk.setLength(123L); Assert.assertEquals("Unexpected value for length() after adding one SegmentChunk.", chunk.getStartOffset() + chunk.getLength(), h.length()); AssertExtensions.assertListEquals("Unexpected contents for chunks().", Collections.singletonList(chunk), h.chunks(), Object::equals); Assert.assertEquals("Unexpected number of registered SegmentChunks.", 2, h.chunks().size()); Assert.assertEquals("Unexpected value for length() after adding two SegmentChunk.", chunk2.getStartOffset() + chunk2.getLength(), h.length()); Assert.assertEquals("Unexpected lastChunk.", chunk2, h.lastChunk()); h.lastChunk().markInexistent();