} catch (Exception e) { if (consumer != null) { log.warn("Error executing interceptor beforeConsume callback topic: {} consumerName: {}", consumer.getTopic(), consumer.getConsumerName(), e); } else { log.warn("Error executing interceptor beforeConsume callback", e);
@Override public void received(Consumer<T> consumer, Message<T> msg) { readerListener.received(ReaderImpl.this, msg); consumer.acknowledgeCumulativeAsync(msg); }
protected void testPulsarFunctionality(String pulsarBrokerUrl) throws Exception { try ( PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarBrokerUrl) .build(); Consumer consumer = client.newConsumer() .topic(TEST_TOPIC) .subscriptionName("test-subs") .subscribe(); Producer<byte[]> producer = client.newProducer() .topic(TEST_TOPIC) .create() ) { producer.send("test containers".getBytes()); CompletableFuture<Message> future = consumer.receiveAsync(); Message message = future.get(5, TimeUnit.SECONDS); assertThat(new String(message.getData())) .isEqualTo("test containers"); } }
@Override public void received(Consumer<T> consumer, Message<T> message) { String topicName; // If more than one topics are being read than the Message return by the consumer will be TopicMessageImpl // If there is only topic being read then the Message returned by the consumer wil be MessageImpl if (message instanceof TopicMessageImpl) { topicName = ((TopicMessageImpl<?>) message).getTopicName(); } else { topicName = consumer.getTopic(); } Record<T> record = PulsarRecord.<T>builder() .message(message) .topicName(topicName) .ackFunction(() -> { if (pulsarSourceConfig .getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { consumer.acknowledgeCumulativeAsync(message); } else { consumer.acknowledgeAsync(message); } }).failFunction(() -> { if (pulsarSourceConfig.getProcessingGuarantees() == FunctionConfig.ProcessingGuarantees.EFFECTIVELY_ONCE) { throw new RuntimeException("Failed to process message: " + message.getMessageId()); } }) .build(); consume(record); }
@Override protected void acknowledgeIDs(long checkpointId, Set<MessageId> messageIds) { if (consumer == null) { LOG.error("null consumer unable to acknowledge messages"); throw new RuntimeException("null pulsar consumer unable to acknowledge messages"); } if (messageIds.isEmpty()) { LOG.info("no message ids to acknowledge"); return; } Map<String, CompletableFuture<Void>> futures = new HashMap<>(messageIds.size()); for (MessageId id : messageIds) { futures.put(id.toString(), consumer.acknowledgeAsync(id)); } futures.forEach((k, f) -> { try { f.get(); } catch (Exception e) { LOG.error("failed to acknowledge messageId " + k, e); throw new RuntimeException("Messages could not be acknowledged during checkpoint creation.", e); } }); }
public boolean removeConsumer(ConsumerHandler consumer) { final String topicName = consumer.getConsumer().getTopic(); if (topicConsumerMap.containsKey(topicName)) { return topicConsumerMap.get(topicName).remove(consumer); } return false; }
@Override public void close() throws IOException { if (consumer != null) { if (!this.service.removeConsumer(this)) { log.warn("[{}] Failed to remove consumer handler", consumer.getTopic()); } consumer.closeAsync().thenAccept(x -> { if (log.isDebugEnabled()) { log.debug("[{}] Closed consumer asynchronously", consumer.getTopic()); } }).exceptionally(exception -> { log.warn("[{}] Failed to close consumer", consumer.getTopic(), exception); return null; }); } }
consumer.receiveAsync().thenAccept(msg -> { if (log.isDebugEnabled()) { log.debug("[{}] [{}] [{}] Got message {}", getSession().getRemoteAddress(), topic, subscription, log.info("[{}/{}] Consumer was closed while receiving msg from broker", consumer.getTopic(), subscription); } else { log.warn("[{}/{}] Error occurred while consumer handler was delivering msg to {}: {}", consumer.getTopic(), subscription, getRemote().getInetSocketAddress().toString(), exception.getMessage());
Message<byte[]> msg = consumer.receive(5, TimeUnit.SECONDS); if (msg == null) { LOG.debug("No message to consume after waiting for 5 seconds."); String output = this.interpretMessage(msg, displayHex); System.out.println(output); consumer.acknowledge(msg);
@Override public void close() throws Exception { consumer.close(); }
@Override public void run(SourceContext<T> context) throws Exception { Message message; while (isRunning) { message = consumer.receive(messageReceiveTimeoutMs, TimeUnit.MILLISECONDS); if (message == null) { continue; } if (isCheckpointingEnabled) { emitCheckpointing(context, message); } else { emitAutoAcking(context, message); } } }
public void start() throws Exception { Producer producer = getNewProducer(); final Consumer consumer = consumerFuture.get(); while (!stop.get()) { final MutableBoolean wellnessFlag = new MutableBoolean(); final Function<Throwable, ? extends MessageId> exceptionHandler = e -> { // Unset the well flag in the case of an exception so we can // try to get a new Producer. wellnessFlag.value = false; return null; }; while (!stop.get() && wellnessFlag.value) { producer.sendAsync(payload.get()).exceptionally(exceptionHandler); rateLimiter.acquire(); } producer.closeAsync(); if (!stop.get()) { // The Producer failed due to an exception: attempt to get // another producer. producer = getNewProducer(); } else { // We are finished: close the consumer. consumer.closeAsync(); } } } }
@Override public void onWebSocketText(String message) { super.onWebSocketText(message); // We should have received an ack MessageId msgId; try { ConsumerAck ack = ObjectMapperFactory.getThreadLocal().readValue(message, ConsumerAck.class); msgId = MessageId.fromByteArrayWithTopic(Base64.getDecoder().decode(ack.messageId), topic); } catch (IOException e) { log.warn("Failed to deserialize message id: {}", message, e); close(WebSocketError.FailedToDeserializeFromJSON); return; } consumer.acknowledgeAsync(msgId).thenAccept(consumer -> numMsgsAcked.increment()); int pending = pendingMessages.getAndDecrement(); if (pending >= maxPendingMessages) { // Resume delivery receiveMessage(); } }
public boolean addConsumer(ConsumerHandler consumer) { return topicConsumerMap .computeIfAbsent(consumer.getConsumer().getTopic(), topic -> new ConcurrentOpenHashSet<>()) .add(consumer); }
@Override public void close() throws Exception { inputConsumers.forEach(consumer -> { try { consumer.close(); } catch (PulsarClientException e) { } }); }
consumer.acknowledgeAsync(msg); };
public boolean removeReader(ReaderHandler reader) { final String topicName = reader.getConsumer().getTopic(); if (topicReaderMap.containsKey(topicName)) { return topicReaderMap.get(topicName).remove(reader); } return false; }
} catch (Exception e) { if (consumer != null) { log.warn("Error executing interceptor beforeConsume callback topic: {} consumerName: {}", consumer.getTopic(), consumer.getConsumerName(), e); } else { log.warn("Error executing interceptor beforeConsume callback", e);
@Override protected CompletableFuture<Void> doAcknowledge(MessageId messageId, AckType ackType, Map<String,Long> properties) { checkArgument(messageId instanceof TopicMessageIdImpl); TopicMessageIdImpl topicMessageId = (TopicMessageIdImpl) messageId; if (getState() != State.Ready) { return FutureUtil.failedFuture(new PulsarClientException("Consumer already closed")); } if (ackType == AckType.Cumulative) { Consumer individualConsumer = consumers.get(topicMessageId.getTopicPartitionName()); if (individualConsumer != null) { MessageId innerId = topicMessageId.getInnerMessageId(); return individualConsumer.acknowledgeCumulativeAsync(innerId); } else { return FutureUtil.failedFuture(new PulsarClientException.NotConnectedException()); } } else { ConsumerImpl<T> consumer = consumers.get(topicMessageId.getTopicPartitionName()); MessageId innerId = topicMessageId.getInnerMessageId(); return consumer.doAcknowledge(innerId, ackType, properties) .thenRun(() -> unAckedMessageTracker.remove(topicMessageId)); } }
@Override public CompletableFuture<BenchmarkConsumer> createConsumer(String topic, String subscriptionName, ConsumerCallback consumerCallback) { return client.newConsumer().subscriptionType(SubscriptionType.Failover).messageListener((consumer, msg) -> { consumerCallback.messageReceived(msg.getData(), msg.getPublishTime()); consumer.acknowledgeAsync(msg); }).topic(topic).subscriptionName(subscriptionName).subscribeAsync() .thenApply(consumer -> new PulsarBenchmarkConsumer(consumer)); }