protected void testPulsarFunctionality(String pulsarBrokerUrl) throws Exception { try ( PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarBrokerUrl) .build(); Consumer consumer = client.newConsumer() .topic(TEST_TOPIC) .subscriptionName("test-subs") .subscribe(); Producer<byte[]> producer = client.newProducer() .topic(TEST_TOPIC) .create() ) { producer.send("test containers".getBytes()); CompletableFuture<Message> future = consumer.receiveAsync(); Message message = future.get(5, TimeUnit.SECONDS); assertThat(new String(message.getData())) .isEqualTo("test containers"); } }
try { deadLetterProducer = client.newProducer(schema) .topic(this.deadLetterPolicy.getDeadLetterTopic()) .blockIfQueueFull(false) .create(); } catch (Exception e) { log.error("Create dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), e);
.sendTimeout(0, TimeUnit.SECONDS) // .compressionType(arguments.compression) // .maxPendingMessages(arguments.maxOutstanding) // .messageRoutingMode(MessageRoutingMode.RoundRobinPartition); producerBuilder.enableBatching(false); } else { long batchTimeUsec = (long) (arguments.batchTimeMillis * 1000); producerBuilder.batchingMaxPublishDelay(batchTimeUsec, TimeUnit.MICROSECONDS) .enableBatching(true); producerBuilder.blockIfQueueFull(true); producerBuilder.addEncryptionKey(arguments.encKeyName); byte[] pKey = Files.readAllBytes(Paths.get(arguments.encKeyFile)); EncKeyReader keyReader = new EncKeyReader(pKey); producerBuilder.cryptoKeyReader(keyReader); futures.add(producerBuilder.clone().topic(topic).createAsync());
@Override public void start() { this.state = State.STARTING; try { ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer() .topic(logTopic) .producerName(fqn) .blockIfQueueFull(false) .enableBatching(true) .batchingMaxPublishDelay(10, TimeUnit.MILLISECONDS); producer = producerBuilder.create(); } catch (Exception e) { throw new RuntimeException("Error starting LogTopic Producer", e); } this.state = State.STARTED; }
private ProducerBuilder<byte[]> getProducerBuilder(PulsarClient client) { ProducerBuilder<byte[]> builder = client.newProducer() .enableBatching(false) .messageRoutingMode(MessageRoutingMode.SinglePartition); builder.blockIfQueueFull(false); builder.producerName(queryParams.get("producerName")); builder.initialSequenceId(Long.parseLong("initialSequenceId")); builder.hashingScheme(HashingScheme.valueOf(queryParams.get("hashingScheme"))); builder.sendTimeout(Integer.parseInt(queryParams.get("sendTimeoutMillis")), TimeUnit.MILLISECONDS); builder.enableBatching(Boolean.parseBoolean(queryParams.get("batchingEnabled"))); builder.batchingMaxMessages(Integer.parseInt(queryParams.get("batchingMaxMessages"))); builder.maxPendingMessages(Integer.parseInt(queryParams.get("maxPendingMessages"))); builder.batchingMaxPublishDelay(Integer.parseInt(queryParams.get("batchingMaxPublishDelay")), TimeUnit.MILLISECONDS); MessageRoutingMode routingMode = MessageRoutingMode.valueOf(queryParams.get("messageRoutingMode")); if (!MessageRoutingMode.CustomPartition.equals(routingMode)) { builder.messageRoutingMode(routingMode); checkArgument(Enums.getIfPresent(CompressionType.class, queryParams.get("compressionType")).isPresent(), "Invalid compressionType %s", queryParams.get("compressionType"));
public AbstractReplicator(String topicName, String replicatorPrefix, String localCluster, String remoteCluster, BrokerService brokerService) throws NamingException { validatePartitionedTopic(topicName, brokerService); this.brokerService = brokerService; this.topicName = topicName; this.replicatorPrefix = replicatorPrefix; this.localCluster = localCluster.intern(); this.remoteCluster = remoteCluster.intern(); this.client = (PulsarClientImpl) brokerService.getReplicationClient(remoteCluster); this.producer = null; this.producerQueueSize = brokerService.pulsar().getConfiguration().getReplicationProducerQueueSize(); this.producerBuilder = client.newProducer() // .topic(topicName) .messageRoutingMode(MessageRoutingMode.SinglePartition) .enableBatching(false) .sendTimeout(0, TimeUnit.SECONDS) // .maxPendingMessages(producerQueueSize) // .producerName(getReplicatorName(replicatorPrefix, localCluster)); STATE_UPDATER.set(this, State.Stopped); }
producerBuilder = client.newProducer().enableBatching(config.producer.batchingEnabled) .batchingMaxPublishDelay(config.producer.batchingMaxPublishDelayMs, TimeUnit.MILLISECONDS) .blockIfQueueFull(config.producer.blockIfQueueFull) .maxPendingMessages(config.producer.pendingQueueSize);
public ContextImpl(InstanceConfig config, Logger logger, PulsarClient client, List<String> inputTopics) { this.config = config; this.logger = logger; this.currentAccumulatedMetrics = new ConcurrentHashMap<>(); this.accumulatedMetrics = new ConcurrentHashMap<>(); this.publishProducers = new HashMap<>(); this.inputTopics = inputTopics; this.topicSchema = new TopicSchema(client); this.producerBuilder = (ProducerBuilderImpl<?>) client.newProducer().blockIfQueueFull(true).enableBatching(true) .batchingMaxPublishDelay(1, TimeUnit.MILLISECONDS); if (config.getFunctionDetails().getUserConfig().isEmpty()) { userConfigs = new HashMap<>(); } else { userConfigs = new Gson().fromJson(config.getFunctionDetails().getUserConfig(), new TypeToken<Map<String, Object>>() { }.getType()); } }
@Override public CompletableFuture<BenchmarkProducer> createProducer(String topic) { return producerBuilder.topic(topic).createAsync() .thenApply(pulsarProducer -> new PulsarBenchmarkProducer(pulsarProducer)); }
public NonPersistentReplicator(NonPersistentTopic topic, String localCluster, String remoteCluster, BrokerService brokerService) throws NamingException { super(topic.getName(), topic.replicatorPrefix, localCluster, remoteCluster, brokerService); producerBuilder.blockIfQueueFull(false); startProducer(); }
producerBuilder.createAsync().thenAccept(producer -> { readEntries(producer); }).exceptionally(ex -> {
@SuppressWarnings("unchecked") public <O> CompletableFuture<Void> publish(String topicName, O object, Schema<O> schema) { Producer<O> producer = (Producer<O>) publishProducers.get(topicName); if (producer == null) { try { Producer<O> newProducer = ((ProducerBuilderImpl<O>) producerBuilder.clone()) .schema(schema).topic(topicName).create(); Producer<O> existingProducer = (Producer<O>) publishProducers.putIfAbsent(topicName, newProducer); if (existingProducer != null) { // The value in the map was not updated after the concurrent put newProducer.close(); producer = existingProducer; } else { producer = newProducer; } } catch (PulsarClientException e) { logger.error("Failed to create Producer while doing user publish", e); return FutureUtil.failedFuture(e); } } return producer.sendAsync(object).thenApply(msgId -> null); }
try { deadLetterProducer = client.newProducer(schema) .topic(this.deadLetterPolicy.getDeadLetterTopic()) .blockIfQueueFull(false) .create(); } catch (Exception e) { log.error("Create dead letter producer exception with topic: {}", deadLetterPolicy.getDeadLetterTopic(), e);
@Override public void start() { try { client = PulsarClient.builder() .serviceUrl(serviceUrl) .build(); log.info("Successfully created pulsar client to {}", serviceUrl); producer = client.newProducer(Schema.BYTES) .topic(topic) .create(); log.info("Successfully created producer to produce updates to topic {}", topic); reader = client.newReader(Schema.BYTES) .topic(topic) .startMessageId(MessageId.earliest) .create(); log.info("Successfully created reader to replay updates from topic {}", topic); CompletableFuture<Void> endFuture = new CompletableFuture<>(); readToEnd(endFuture); endFuture.join(); } catch (PulsarClientException e) { log.error("Failed to create pulsar client to cluster at {}", serviceUrl, e); throw new RuntimeException("Failed to create pulsar client to cluster at " + serviceUrl, e); } }
public ProducerHandler(WebSocketService service, HttpServletRequest request, ServletUpgradeResponse response) { super(service, request, response); this.numMsgsSent = new LongAdder(); this.numBytesSent = new LongAdder(); this.numMsgsFailed = new LongAdder(); this.publishLatencyStatsUSec = new StatsBuckets(ENTRY_LATENCY_BUCKETS_USEC); if (!checkAuth(response)) { return; } try { this.producer = getProducerBuilder(service.getPulsarClient()).topic(topic.toString()).create(); if (!this.service.addProducer(this)) { log.warn("[{}:{}] Failed to add producer handler for topic {}", request.getRemoteAddr(), request.getRemotePort(), topic); } } catch (Exception e) { log.warn("[{}:{}] Failed in creating producer on topic {}: {}", request.getRemoteAddr(), request.getRemotePort(), topic, e.getMessage()); try { response.sendError(getErrorCode(e), getErrorMessage(e)); } catch (IOException e1) { log.warn("[{}:{}] Failed to send error: {}", request.getRemoteAddr(), request.getRemotePort(), e1.getMessage(), e1); } } }