public static ByteBuf serializeSingleMessageInBatchWithPayload( PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder, ByteBuf payload, ByteBuf batchBuffer) { int payLoadSize = payload.readableBytes(); PulsarApi.SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.setPayloadSize(payLoadSize) .build(); // serialize meta-data size, meta-data and payload for single message in batch int singleMsgMetadataSize = singleMessageMetadata.getSerializedSize(); try { batchBuffer.writeInt(singleMsgMetadataSize); ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(batchBuffer); singleMessageMetadata.writeTo(outStream); singleMessageMetadata.recycle(); outStream.recycle(); } catch (IOException e) { throw new RuntimeException(e); } return batchBuffer.writeBytes(payload); }
messageId.getEntryId(), getPartitionIndex(), i, acker); final MessageImpl<T> message = new MessageImpl<>(topicName.toString(), batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, createEncryptionContext(msgMetadata), cnx, schema, redeliveryCount); if (possibleToDeadLetter != null) {
public static ByteBuf deSerializeSingleMessageInBatch(ByteBuf uncompressedPayload, PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder, int index, int batchSize) throws IOException { int singleMetaSize = (int) uncompressedPayload.readUnsignedInt(); int writerIndex = uncompressedPayload.writerIndex(); int beginIndex = uncompressedPayload.readerIndex() + singleMetaSize; uncompressedPayload.writerIndex(beginIndex); ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(uncompressedPayload); PulsarApi.SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.mergeFrom(stream, null) .build(); int singleMessagePayloadSize = singleMessageMetadata.getPayloadSize(); uncompressedPayload.markReaderIndex(); ByteBuf singleMessagePayload = uncompressedPayload.slice(uncompressedPayload.readerIndex(), singleMessagePayloadSize); singleMessagePayload.retain(); uncompressedPayload.writerIndex(writerIndex); uncompressedPayload.resetReaderIndex(); // reader now points to beginning of payload read; so move it past message payload just read if (index < batchSize) { uncompressedPayload.readerIndex(uncompressedPayload.readerIndex() + singleMessagePayloadSize); } return singleMessagePayload; }
messageId.getEntryId(), getPartitionIndex(), i, acker); final MessageImpl<T> message = new MessageImpl<>(topicName.toString(), batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, createEncryptionContext(msgMetadata), cnx, schema); if (possibleToDeadLetter != null) {
public static void receiveIndividualMessagesFromBatch(String topic, MessageMetadata msgMetadata, ByteBuf uncompressedPayload, MessageIdData messageId, ClientCnx cnx, int partitionIndex, MessageProcessor processor) { int batchSize = msgMetadata.getNumMessagesInBatch(); try { for (int i = 0; i < batchSize; ++i) { PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadataBuilder, i, batchSize); if (singleMessageMetadataBuilder.getCompactedOut()) { // message has been compacted out, so don't send to the user singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); continue; } BatchMessageIdImpl batchMessageIdImpl = new BatchMessageIdImpl(messageId.getLedgerId(), messageId.getEntryId(), partitionIndex, i, null); final MessageImpl<?> message = new MessageImpl<>( topic, batchMessageIdImpl, msgMetadata, singleMessageMetadataBuilder.build(), singleMessagePayload, Optional.empty(), cnx, null); processor.process(batchMessageIdImpl, message, singleMessagePayload); singleMessagePayload.release(); singleMessageMetadataBuilder.recycle(); } } catch (IOException e) { log.warn("Unable to obtain messages in batch", e); } }
private List<Message<byte[]>> getIndividualMsgsFromBatch(String topic, String msgId, byte[] data, Map<String, String> properties) { List<Message<byte[]>> ret = new ArrayList<>(); int batchSize = Integer.parseInt(properties.get(BATCH_HEADER)); for (int i = 0; i < batchSize; i++) { String batchMsgId = msgId + ":" + i; PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); ByteBuf buf = Unpooled.wrappedBuffer(data); try { ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(buf, singleMessageMetadataBuilder, i, batchSize); SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.build(); if (singleMessageMetadata.getPropertiesCount() > 0) { for (KeyValue entry : singleMessageMetadata.getPropertiesList()) { properties.put(entry.getKey(), entry.getValue()); } } ret.add(new MessageImpl<>(topic, batchMsgId, properties, singleMessagePayload, Schema.BYTES)); } catch (Exception ex) { log.error("Exception occured while trying to get BatchMsgId: {}", batchMsgId, ex); } buf.release(); singleMessageMetadataBuilder.recycle(); } return ret; }
public static ByteBuf serializeSingleMessageInBatchWithPayload( PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder, ByteBuf payload, ByteBuf batchBuffer) { int payLoadSize = payload.readableBytes(); PulsarApi.SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.setPayloadSize(payLoadSize) .build(); // serialize meta-data size, meta-data and payload for single message in batch int singleMsgMetadataSize = singleMessageMetadata.getSerializedSize(); try { batchBuffer.writeInt(singleMsgMetadataSize); ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(batchBuffer); singleMessageMetadata.writeTo(outStream); singleMessageMetadata.recycle(); outStream.recycle(); } catch (IOException e) { throw new RuntimeException(e); } return batchBuffer.writeBytes(payload); }