private static void dissect(final DataHeaderFlyweight msg, final StringBuilder builder) { builder .append(msg.headerType() == HeaderFlyweight.HDR_TYPE_PAD ? "PAD" : "DATA") .append(' ') .append(msg.flags()) .append(" len ") .append(msg.frameLength()) .append(' ') .append(msg.sessionId()) .append(':') .append(msg.streamId()) .append(':') .append(msg.termId()) .append(" @") .append(msg.termOffset()); }
public NetworkPublicationThreadLocals() { final ByteBuffer byteBuffer = BufferUtil.allocateDirectAligned(192, BitUtil.CACHE_LINE_LENGTH); byteBuffer.limit(DataHeaderFlyweight.HEADER_LENGTH); heartbeatBuffer = byteBuffer.slice(); dataHeader = new DataHeaderFlyweight(heartbeatBuffer); byteBuffer.limit(64 + SetupFlyweight.HEADER_LENGTH).position(64); setupBuffer = byteBuffer.slice(); setupHeader = new SetupFlyweight(setupBuffer); byteBuffer.limit(128 + RttMeasurementFlyweight.HEADER_LENGTH).position(128); rttMeasurementBuffer = byteBuffer.slice(); rttMeasurementHeader = new RttMeasurementFlyweight(rttMeasurementBuffer); dataHeader .version(HeaderFlyweight.CURRENT_VERSION) .flags((byte)DataHeaderFlyweight.BEGIN_AND_END_FLAGS) .headerType(HeaderFlyweight.HDR_TYPE_DATA) .frameLength(0); setupHeader .version(HeaderFlyweight.CURRENT_VERSION) .headerType(HeaderFlyweight.HDR_TYPE_SETUP) .frameLength(SetupFlyweight.HEADER_LENGTH); rttMeasurementHeader .version(HeaderFlyweight.CURRENT_VERSION) .headerType(HeaderFlyweight.HDR_TYPE_RTTM) .frameLength(RttMeasurementFlyweight.HEADER_LENGTH); }
private void fillDataFrame(final DataHeaderFlyweight header, final int termOffset, final byte[] payload) { header.wrap(dataBuffer); header .termOffset(termOffset) .termId(ACTIVE_TERM_ID) .streamId(STREAM_ID) .sessionId(SESSION_ID) .frameLength(DataHeaderFlyweight.HEADER_LENGTH + payload.length) .headerType(HeaderFlyweight.HDR_TYPE_DATA) .flags(DataHeaderFlyweight.BEGIN_AND_END_FLAGS) .version(HeaderFlyweight.CURRENT_VERSION); if (0 < payload.length) { dataBuffer.putBytes(header.dataOffset(), payload); } }
static boolean isInvalidHeader( final UnsafeBuffer buffer, final int streamId, final int termId, final int termOffset) { return DataHeaderFlyweight.termOffset(buffer, 0) != termOffset || DataHeaderFlyweight.termId(buffer, 0) != termId || DataHeaderFlyweight.streamId(buffer, 0) != streamId; } }
private int heartbeatMessageCheck( final long nowNs, final int activeTermId, final int termOffset, final boolean isEndOfStream) { int bytesSent = 0; if ((timeOfLastSendOrHeartbeatNs + PUBLICATION_HEARTBEAT_TIMEOUT_NS) - nowNs < 0) { heartbeatBuffer.clear(); heartbeatDataHeader .sessionId(sessionId) .streamId(streamId) .termId(activeTermId) .termOffset(termOffset) .flags((byte)(isEndOfStream ? BEGIN_END_AND_EOS_FLAGS : BEGIN_AND_END_FLAGS)); bytesSent = channelEndpoint.send(heartbeatBuffer); if (DataHeaderFlyweight.HEADER_LENGTH != bytesSent) { shortSends.increment(); } timeOfLastSendOrHeartbeatNs = nowNs; heartbeatsSent.incrementOrdered(); } return bytesSent; }
private void initPublicationMetadata( final int sessionId, final int streamId, final int initialTermId, final long registrationId, final PublicationParams params, final RawLog rawLog) { final UnsafeBuffer logMetaData = rawLog.metaData(); defaultDataHeader.sessionId(sessionId).streamId(streamId).termId(initialTermId); storeDefaultFrameHeader(logMetaData, defaultDataHeader); initialTermId(logMetaData, initialTermId); mtuLength(logMetaData, params.mtuLength); termLength(logMetaData, rawLog.termLength()); pageSize(logMetaData, ctx.filePageSize()); correlationId(logMetaData, registrationId); endOfStreamPosition(logMetaData, Long.MAX_VALUE); initialisePositionCounters(initialTermId, params, logMetaData); }
private void recordFragment( final RecordingWriter recordingWriter, final UnsafeBuffer buffer, final DataHeaderFlyweight headerFlyweight, final Header header, final int message, final byte flags, final int type) { final int offset = INITIAL_TERM_OFFSET + message * FRAME_LENGTH; headerFlyweight.wrap(buffer, offset, HEADER_LENGTH); headerFlyweight .streamId(STREAM_ID) .sessionId(SESSION_ID) .termOffset(offset) .termId(INITIAL_TERM_ID) .reservedValue(message) .headerType(type) .flags(flags) .frameLength(FRAME_LENGTH); buffer.setMemory( offset + HEADER_LENGTH, FRAME_LENGTH - HEADER_LENGTH, (byte)message); header.offset(offset); recordingWriter.onBlock(buffer, offset, FRAME_LENGTH, SESSION_ID, INITIAL_TERM_ID); recordingPosition += FRAME_LENGTH; }
sendChannelEndpoint.registerForRead(controlTransportPoller); encodeDataHeader.wrap(buffer); encodeDataHeader .version(HeaderFlyweight.CURRENT_VERSION) .flags(DataHeaderFlyweight.BEGIN_AND_END_FLAGS) .headerType(HeaderFlyweight.HDR_TYPE_DATA) .frameLength(FRAME_LENGTH); encodeDataHeader .sessionId(SESSION_ID) .streamId(STREAM_ID) .termId(TERM_ID); encodeDataHeader.wrap(buffer, alignedFrameLength, buffer.capacity() - alignedFrameLength); encodeDataHeader .version(HeaderFlyweight.CURRENT_VERSION) .flags(DataHeaderFlyweight.BEGIN_AND_END_FLAGS) .headerType(HeaderFlyweight.HDR_TYPE_DATA) .frameLength(24); encodeDataHeader .sessionId(SESSION_ID) .streamId(STREAM_ID) .termId(TERM_ID);
@Before public void before() throws Exception { when(mockPosition.getWeak()).then((invocation) -> positionLong); when(mockPosition.get()).then((invocation) -> positionLong); doAnswer( (invocation) -> { positionLong = invocation.getArgument(0); return null; }) .when(mockPosition).setOrdered(anyLong()); termFile = File.createTempFile("test.rec", "sourceIdentity"); mockLogBufferChannel = FileChannel.open(termFile.toPath(), CREATE, READ, WRITE); mockLogBufferMapped = new UnsafeBuffer( mockLogBufferChannel.map(FileChannel.MapMode.READ_WRITE, 0, TERM_BUFFER_LENGTH)); final DataHeaderFlyweight headerFlyweight = new DataHeaderFlyweight(); headerFlyweight.wrap(mockLogBufferMapped, TERM_OFFSET, DataHeaderFlyweight.HEADER_LENGTH); headerFlyweight .termOffset(TERM_OFFSET) .sessionId(SESSION_ID) .streamId(STREAM_ID) .headerType(DataHeaderFlyweight.HDR_TYPE_DATA) .frameLength(RECORDED_BLOCK_LENGTH); context = new Archive.Context() .segmentFileLength(SEGMENT_LENGTH) .archiveDir(archiveDir) .catalog(mockCatalog) .epochClock(epochClock); }
private void insertPaddingFrame(final int activeTermId, final int termOffset) { dataHeader .termId(INITIAL_TERM_ID) .streamId(STREAM_ID) .sessionId(SESSION_ID) .frameLength(TERM_BUFFER_LENGTH - termOffset) .headerType(HeaderFlyweight.HDR_TYPE_PAD) .flags(DataHeaderFlyweight.BEGIN_AND_END_FLAGS) .version(HeaderFlyweight.CURRENT_VERSION); final int activeIndex = indexByTerm(INITIAL_TERM_ID, activeTermId); TermRebuilder.insert(termBuffers[activeIndex], termOffset, rcvBuffer, TERM_BUFFER_LENGTH - termOffset); }
if (HEADER_FLYWEIGHT.frameLength() != 0) if (HEADER_FLYWEIGHT.sessionId() != decoder.sessionId()) HEADER_FLYWEIGHT.sessionId() + " (expected=" + decoder.sessionId() + ")"); return true; if (HEADER_FLYWEIGHT.streamId() != decoder.streamId()) HEADER_FLYWEIGHT.streamId() + " (expected=" + decoder.streamId() + ")"); return true; position += BitUtil.align(HEADER_FLYWEIGHT.frameLength(), FrameDescriptor.FRAME_ALIGNMENT); while (HEADER_FLYWEIGHT.frameLength() != 0);
@Test public void shouldNotSendSetupFrameAfterReceivingStatusMessage() { final StatusMessageFlyweight msg = mock(StatusMessageFlyweight.class); when(msg.consumptionTermId()).thenReturn(INITIAL_TERM_ID); when(msg.consumptionTermOffset()).thenReturn(0); when(msg.receiverWindowLength()).thenReturn(0); publication.onStatusMessage(msg, rcvAddress); // publication.senderPositionLimit(flowControl.onStatusMessage(msg, rcvAddress, )); sender.doWork(); assertThat(receivedFrames.size(), is(1)); receivedFrames.remove(); currentTimestamp += Configuration.PUBLICATION_SETUP_TIMEOUT_NS + 10; sender.doWork(); assertThat(receivedFrames.size(), is(1)); dataHeader.wrap(receivedFrames.remove()); assertThat(dataHeader.headerType(), is(HeaderFlyweight.HDR_TYPE_DATA)); // heartbeat assertThat(dataHeader.frameLength(), is(0)); assertThat(dataHeader.termOffset(), is(offsetOfMessage(1))); }
@Test public void shouldSendLastDataFrameAsHeartbeatWhenIdle() { final StatusMessageFlyweight msg = mock(StatusMessageFlyweight.class); when(msg.consumptionTermId()).thenReturn(INITIAL_TERM_ID); when(msg.consumptionTermOffset()).thenReturn(0); when(msg.receiverWindowLength()).thenReturn(ALIGNED_FRAME_LENGTH); publication.onStatusMessage(msg, rcvAddress); final UnsafeBuffer buffer = new UnsafeBuffer(ByteBuffer.allocateDirect(PAYLOAD.length)); buffer.putBytes(0, PAYLOAD); termAppenders[0].appendUnfragmentedMessage(headerWriter, buffer, 0, PAYLOAD.length, null, INITIAL_TERM_ID); sender.doWork(); assertThat(receivedFrames.size(), is(2)); // should send ticks receivedFrames.remove(); // skip setup & data frame receivedFrames.remove(); currentTimestamp += Configuration.PUBLICATION_HEARTBEAT_TIMEOUT_NS - 1; sender.doWork(); assertThat(receivedFrames.size(), is(0)); // should not send yet currentTimestamp += 10; sender.doWork(); assertThat(receivedFrames.size(), greaterThanOrEqualTo(1)); // should send ticks dataHeader.wrap(receivedFrames.remove()); assertThat(dataHeader.frameLength(), is(0)); assertThat(dataHeader.termOffset(), is(offsetOfMessage(2))); }
/** * Dump the contents of a segment file to a {@link PrintStream}. * * @param out for the dumped contents. * @param messageDumpLimit for the number of bytes per message fragment to dump. * @param buffer the wraps the segment file. */ public static void dumpSegment(final PrintStream out, final int messageDumpLimit, final UnsafeBuffer buffer) { final DataHeaderFlyweight dataHeaderFlyweight = new DataHeaderFlyweight(); final int length = buffer.capacity(); int offset = 0; while (offset < length) { dataHeaderFlyweight.wrap(buffer, offset, length - offset); out.println(offset + ": " + dataHeaderFlyweight.toString()); final int frameLength = dataHeaderFlyweight.frameLength(); if (frameLength < DataHeaderFlyweight.HEADER_LENGTH) { break; } final int limit = min(frameLength - HEADER_LENGTH, messageDumpLimit); out.println(LogInspector.formatBytes(buffer, offset + HEADER_LENGTH, limit)); offset += BitUtil.align(frameLength, FrameDescriptor.FRAME_ALIGNMENT); } } }
private void createSegmentFile(final long newRecordingId) throws IOException { final File segmentFile = new File(archiveDir, segmentFileName(newRecordingId, 0)); try (FileChannel log = FileChannel.open(segmentFile.toPath(), READ, WRITE, CREATE)) { final ByteBuffer bb = ByteBuffer.allocateDirect(HEADER_LENGTH); final DataHeaderFlyweight flyweight = new DataHeaderFlyweight(bb); flyweight.frameLength(PAGE_SIZE - HEADER_LENGTH); log.write(bb); bb.clear(); flyweight.frameLength(128); log.write(bb, PAGE_SIZE - HEADER_LENGTH); bb.clear(); flyweight.frameLength(0); log.write(bb, PAGE_SIZE - HEADER_LENGTH + 128); } } }
@Test public void shouldNotOverwriteExistingFrame() { final int gapOffset = 0; final int gapLength = 64; dataFlyweight.frameLength(32); assertFalse(TermGapFiller.tryFillGap(metaDataBuffer, termBuffer, TERM_ID, gapOffset, gapLength)); }
public LossDetectorTest() { lossHandler = mock(LossHandler.class); lossDetector = new LossDetector(DELAY_GENERATOR, lossHandler); dataHeader.wrap(rcvBuffer); }
public DataTransportPoller() { byteBuffer = NetworkUtil.allocateDirectAlignedAndPadded( Configuration.MAX_UDP_PAYLOAD_LENGTH, CACHE_LINE_LENGTH * 2); unsafeBuffer = new UnsafeBuffer(byteBuffer); dataMessage = new DataHeaderFlyweight(unsafeBuffer); setupMessage = new SetupFlyweight(unsafeBuffer); rttMeasurement = new RttMeasurementFlyweight(unsafeBuffer); }
final int transportIndex) final int streamId = header.streamId(); final StreamInterest streamInterest = streamInterestByIdMap.get(streamId); final int sessionId = header.sessionId(); final int termId = header.termId(); final SessionInterest sessionInterest = streamInterest.sessionInterestByIdMap.get(sessionId); termId, header.termOffset(), buffer, length, transportIndex, srcAddress); else if (!DataHeaderFlyweight.isEndOfStream(buffer))
private RawLog newPublicationImageLog( final int sessionId, final int streamId, final int initialTermId, final int termBufferLength, final boolean isSparse, final int senderMtuLength, final UdpChannel udpChannel, final long correlationId) { final RawLog rawLog = rawLogFactory.newNetworkedImage( udpChannel.canonicalForm(), sessionId, streamId, correlationId, termBufferLength, isSparse); final UnsafeBuffer logMetaData = rawLog.metaData(); defaultDataHeader.sessionId(sessionId).streamId(streamId).termId(initialTermId); storeDefaultFrameHeader(logMetaData, defaultDataHeader); initialTermId(logMetaData, initialTermId); mtuLength(logMetaData, senderMtuLength); termLength(logMetaData, termBufferLength); pageSize(logMetaData, ctx.filePageSize()); correlationId(logMetaData, correlationId); endOfStreamPosition(logMetaData, Long.MAX_VALUE); return rawLog; }