if (!msg.isReplicated() && msgMetadataBuilder.hasProducerName()) { PulsarClientException.InvalidMessageException invalidMessageException = new PulsarClientException.InvalidMessageException("Cannot re-use the same message"); msgMetadataBuilder.setSchemaVersion(ByteString.copyFrom(schemaVersion.get())); synchronized (this) { long sequenceId; if (!msgMetadataBuilder.hasSequenceId()) { sequenceId = msgIdGeneratorUpdater.getAndIncrement(this); msgMetadataBuilder.setSequenceId(sequenceId); } else { sequenceId = msgMetadataBuilder.getSequenceId(); if (!msgMetadataBuilder.hasPublishTime()) { msgMetadataBuilder.setPublishTime(System.currentTimeMillis()); checkArgument(!msgMetadataBuilder.hasProducerName()); msgMetadataBuilder.setProducerName(producerName); msgMetadataBuilder.setCompression( CompressionCodecProvider.convertToWireProtocol(conf.getCompressionType())); msgMetadataBuilder.setUncompressedSize(uncompressedSize); MessageMetadata msgMetadata = msgMetadataBuilder.build(); ByteBufPair cmd = sendMessage(producerId, sequenceId, 1, msgMetadata, encryptedPayload); msgMetadataBuilder.recycle();
kvList.add(KeyValue.newBuilder().setKey(key).setValue(value).build()); }); msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName) .setValue(ByteString.copyFrom(keyInfo.getKey())).addAllMetadata(kvList).build()); } else { msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName) .setValue(ByteString.copyFrom(keyInfo.getKey())).build()); msgMetadata.setEncryptionParam(ByteString.copyFrom(iv));
@Override public TypedMessageBuilder<T> properties(Map<String, String> properties) { for (Map.Entry<String, String> entry : properties.entrySet()) { checkArgument(entry.getKey() != null, "Need Non-Null key"); checkArgument(entry.getValue() != null, "Need Non-Null value for key: " + entry.getKey()); msgMetadataBuilder .addProperties(KeyValue.newBuilder().setKey(entry.getKey()).setValue(entry.getValue()).build()); } return this; }
org.apache.pulsar.common.api.proto.PulsarApi.KeyValue.Builder subBuilder = org.apache.pulsar.common.api.proto.PulsarApi.KeyValue.newBuilder(); input.readMessage(subBuilder, extensionRegistry); addProperties(subBuilder.buildPartial()); break; ensureReplicateToIsMutable(); replicateTo_.add(input.readBytes()); break; org.apache.pulsar.common.api.proto.PulsarApi.EncryptionKeys.Builder subBuilder = org.apache.pulsar.common.api.proto.PulsarApi.EncryptionKeys.newBuilder(); input.readMessage(subBuilder, extensionRegistry); addEncryptionKeys(subBuilder.buildPartial()); break;
if (!msg.isReplicated() && msgMetadataBuilder.hasProducerName()) { PulsarClientException.InvalidMessageException invalidMessageException = new PulsarClientException.InvalidMessageException("Cannot re-use the same message"); msgMetadataBuilder.setSchemaVersion(ByteString.copyFrom(schemaVersion.get())); synchronized (this) { long sequenceId; if (!msgMetadataBuilder.hasSequenceId()) { sequenceId = msgIdGeneratorUpdater.getAndIncrement(this); msgMetadataBuilder.setSequenceId(sequenceId); } else { sequenceId = msgMetadataBuilder.getSequenceId(); if (!msgMetadataBuilder.hasPublishTime()) { msgMetadataBuilder.setPublishTime(System.currentTimeMillis()); checkArgument(!msgMetadataBuilder.hasProducerName()); msgMetadataBuilder.setProducerName(producerName); msgMetadataBuilder.setCompression(convertCompressionType(conf.getCompressionType())); msgMetadataBuilder.setUncompressedSize(uncompressedSize); MessageMetadata msgMetadata = msgMetadataBuilder.build(); ByteBufPair cmd = sendMessage(producerId, sequenceId, 1, msgMetadata, encryptedPayload); msgMetadataBuilder.recycle(); msgMetadata.recycle();
metadataBuilder.setUncompressedSize(newUncompressedSize); MessageMetadata newMetadata = metadataBuilder.build(); Optional<RawMessage> result = Optional.of(new RawMessageImpl(msg.getMessageIdData(), metadataAndPayload)); metadataBuilder.recycle(); newMetadata.recycle(); metadataAndPayload.release();
public Builder mergeFrom(org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata other) { if (other == org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata.getDefaultInstance()) return this; if (other.hasProducerName()) { setProducerName(other.getProducerName()); setSequenceId(other.getSequenceId()); setPublishTime(other.getPublishTime()); bitField0_ = (bitField0_ & ~0x00000008); } else { ensurePropertiesIsMutable(); properties_.addAll(other.properties_); setReplicatedFrom(other.getReplicatedFrom()); setPartitionKey(other.getPartitionKey()); bitField0_ = (bitField0_ & ~0x00000040); } else { ensureReplicateToIsMutable(); replicateTo_.addAll(other.replicateTo_); setCompression(other.getCompression()); setUncompressedSize(other.getUncompressedSize()); setNumMessagesInBatch(other.getNumMessagesInBatch()); setEventTime(other.getEventTime());
kvList.add(KeyValue.newBuilder().setKey(key).setValue(value).build()); }); msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName) .setValue(ByteString.copyFrom(keyInfo.getKey())).addAllMetadata(kvList).build()); } else { msgMetadata.addEncryptionKeys(EncryptionKeys.newBuilder().setKey(keyName) .setValue(ByteString.copyFrom(keyInfo.getKey())).build()); msgMetadata.setEncryptionParam(ByteString.copyFrom(iv));
public static ByteBuf serializeSingleMessageInBatchWithPayload(PulsarApi.MessageMetadata.Builder msgBuilder, ByteBuf payload, ByteBuf batchBuffer) { // build single message meta-data PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata .newBuilder(); if (msgBuilder.hasPartitionKey()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder.setPartitionKey(msgBuilder.getPartitionKey()) .setPartitionKeyB64Encoded(msgBuilder.getPartitionKeyB64Encoded()); } if (!msgBuilder.getPropertiesList().isEmpty()) { singleMessageMetadataBuilder = singleMessageMetadataBuilder .addAllProperties(msgBuilder.getPropertiesList()); } if (msgBuilder.hasEventTime()) { singleMessageMetadataBuilder.setEventTime(msgBuilder.getEventTime()); } try { return serializeSingleMessageInBatchWithPayload(singleMessageMetadataBuilder, payload, batchBuffer); } finally { singleMessageMetadataBuilder.recycle(); } }
void add(MessageImpl<?> msg, SendCallback callback) { if (log.isDebugEnabled()) { log.debug("[{}] [{}] add message to batch, num messages in batch so far {}", topicName, producerName, numMessagesInBatch); } if (++numMessagesInBatch == 1) { // some properties are common amongst the different messages in the batch, hence we just pick it up from // the first message sequenceId = Commands.initBatchMessageMetadata(messageMetadata, msg.getMessageBuilder()); this.firstCallback = callback; batchedMessageMetadataAndPayload = PooledByteBufAllocator.DEFAULT .buffer(Math.min(maxBatchSize, MAX_MESSAGE_BATCH_SIZE_BYTES), PulsarDecoder.MaxMessageSize); } if (previousCallback != null) { previousCallback.addCallback(msg, callback); } previousCallback = callback; currentBatchSizeBytes += msg.getDataBuffer().readableBytes(); PulsarApi.MessageMetadata.Builder msgBuilder = msg.getMessageBuilder(); batchedMessageMetadataAndPayload = Commands.serializeSingleMessageInBatchWithPayload(msgBuilder, msg.getDataBuffer(), batchedMessageMetadataAndPayload); messages.add(msg); msgBuilder.recycle(); }
public static MessageMetadata parseMessageMetadata(ByteBuf buffer) { try { // initially reader-index may point to start_of_checksum : increment reader-index to start_of_metadata to parse // metadata skipChecksumIfPresent(buffer); int metadataSize = (int) buffer.readUnsignedInt(); int writerIndex = buffer.writerIndex(); buffer.writerIndex(buffer.readerIndex() + metadataSize); ByteBufCodedInputStream stream = ByteBufCodedInputStream.get(buffer); MessageMetadata.Builder messageMetadataBuilder = MessageMetadata.newBuilder(); MessageMetadata res = messageMetadataBuilder.mergeFrom(stream, null).build(); buffer.writerIndex(writerIndex); messageMetadataBuilder.recycle(); stream.recycle(); return res; } catch (IOException e) { throw new RuntimeException(e); } }
ByteBuf getCompressedBatchMetadataAndPayload() { int uncompressedSize = batchedMessageMetadataAndPayload.readableBytes(); ByteBuf compressedPayload = compressor.encode(batchedMessageMetadataAndPayload); batchedMessageMetadataAndPayload.release(); if (compressionType != PulsarApi.CompressionType.NONE) { messageMetadata.setCompression(compressionType); messageMetadata.setUncompressedSize(uncompressedSize); } // Update the current max batch size using the uncompressed size, which is what we need in any case to // accumulate the batch content maxBatchSize = Math.max(maxBatchSize, uncompressedSize); return compressedPayload; }
@Override public TypedMessageBuilder<T> property(String name, String value) { checkArgument(name != null, "Need Non-Null name"); checkArgument(value != null, "Need Non-Null value for name: " + name); msgMetadataBuilder.addProperties(KeyValue.newBuilder().setKey(name).setValue(value).build()); return this; }
@Override public MessageBuilder<T> setProperties(Map<String, String> properties) { for (Map.Entry<String, String> entry : properties.entrySet()) { msgMetadataBuilder .addProperties(KeyValue.newBuilder().setKey(entry.getKey()).setValue(entry.getValue()).build()); } return this; }
void clear() { messages = Lists.newArrayList(); firstCallback = null; previousCallback = null; messageMetadata.clear(); numMessagesInBatch = 0; currentBatchSizeBytes = 0; sequenceId = -1; batchedMessageMetadataAndPayload = null; }
@Override public TypedMessageBuilder<T> properties(Map<String, String> properties) { for (Map.Entry<String, String> entry : properties.entrySet()) { checkArgument(entry.getKey() != null, "Need Non-Null key"); checkArgument(entry.getValue() != null, "Need Non-Null value for key: " + entry.getKey()); msgMetadataBuilder .addProperties(KeyValue.newBuilder().setKey(entry.getKey()).setValue(entry.getValue()).build()); } return this; }
@Override public TypedMessageBuilder<T> sequenceId(long sequenceId) { checkArgument(sequenceId >= 0); msgMetadataBuilder.setSequenceId(sequenceId); return this; }
@Override public MessageBuilder<T> setSequenceId(long sequenceId) { checkArgument(sequenceId >= 0); msgMetadataBuilder.setSequenceId(sequenceId); return this; }
public void recycle() { clear(); if (handle != null) {RECYCLER.recycle(this, handle);} }
@Override public MessageBuilder<T> setProperty(String name, String value) { msgMetadataBuilder.addProperties(KeyValue.newBuilder().setKey(name).setValue(value).build()); return this; }