for (Producer<T> producer : producers) { if (producer != null) { producer.closeAsync().handle((closed, ex) -> { if (ex != null) { closeFail.compareAndSet(null, ex);
try { for (MessageImpl<T> message : deadLetterMessages) { deadLetterProducer.newMessage() .value(message.getValue()) .properties(message.getProperties()) log.error("Send to dead letter topic exception with topic: {}, messageId: {}", deadLetterProducer.getTopic(), messageId, e);
protected void testPulsarFunctionality(String pulsarBrokerUrl) throws Exception { try ( PulsarClient client = PulsarClient.builder() .serviceUrl(pulsarBrokerUrl) .build(); Consumer consumer = client.newConsumer() .topic(TEST_TOPIC) .subscriptionName("test-subs") .subscribe(); Producer<byte[]> producer = client.newProducer() .topic(TEST_TOPIC) .create() ) { producer.send("test containers".getBytes()); CompletableFuture<Message> future = consumer.receiveAsync(); Message message = future.get(5, TimeUnit.SECONDS); assertThat(new String(message.getData())) .isEqualTo("test containers"); } }
@SuppressWarnings("unchecked") public <O> CompletableFuture<Void> publish(String topicName, O object, Schema<O> schema) { Producer<O> producer = (Producer<O>) publishProducers.get(topicName); if (producer == null) { try { Producer<O> newProducer = ((ProducerBuilderImpl<O>) producerBuilder.clone()) .schema(schema).topic(topicName).create(); Producer<O> existingProducer = (Producer<O>) publishProducers.putIfAbsent(topicName, newProducer); if (existingProducer != null) { // The value in the map was not updated after the concurrent put newProducer.close(); producer = existingProducer; } else { producer = newProducer; } } catch (PulsarClientException e) { logger.error("Failed to create Producer while doing user publish", e); return FutureUtil.failedFuture(e); } } return producer.sendAsync(object).thenApply(msgId -> null); }
@Override public void close() throws Exception { producer.close(); }
public void start() throws Exception { Producer producer = getNewProducer(); final Consumer consumer = consumerFuture.get(); while (!stop.get()) { final MutableBoolean wellnessFlag = new MutableBoolean(); final Function<Throwable, ? extends MessageId> exceptionHandler = e -> { // Unset the well flag in the case of an exception so we can // try to get a new Producer. wellnessFlag.value = false; return null; }; while (!stop.get() && wellnessFlag.value) { producer.sendAsync(payload.get()).exceptionally(exceptionHandler); rateLimiter.acquire(); } producer.closeAsync(); if (!stop.get()) { // The Producer failed due to an exception: attempt to get // another producer. producer = getNewProducer(); } else { // We are finished: close the consumer. consumer.closeAsync(); } } } }
@Override public void logEvent(String event, Map<String, Object> producerConfig) { String output = event; if (flatten) { try { output = jsonUtils.flattenJson(event); } catch (IOException ex) { log.error("Error flattening json. Unable to send event [ " + event + " ]", ex); return; } } if (sync) { try { producer.send(output.getBytes()); } catch (PulsarClientException e) { log.error("Unable to send event to Pulsar", e); } } else { producer.sendAsync(output.getBytes()); } }
@Override public void close() throws IOException { if (producer != null) { if (!this.service.removeProducer(this)) { log.warn("[{}] Failed to remove producer handler", producer.getTopic()); } producer.closeAsync().thenAccept(x -> { if (log.isDebugEnabled()) { log.debug("[{}] Closed producer asynchronously", producer.getTopic()); } }).exceptionally(exception -> { log.warn("[{}] Failed to close producer", producer.getTopic(), exception); return null; }); } }
@Override public void append(LogEvent logEvent) { producer.sendAsync(logEvent.getMessage().getFormattedMessage().getBytes()); }
@Override public Future<Void> set(Map<ByteBuffer, ByteBuffer> values, Callback<Void> callback) { values.forEach((key, value) -> { ByteBuf bb = Unpooled.wrappedBuffer(key); byte[] keyBytes = ByteBufUtil.getBytes(bb); bb = Unpooled.wrappedBuffer(value); byte[] valBytes = ByteBufUtil.getBytes(bb); producer.newMessage() .key(new String(keyBytes, UTF_8)) .value(valBytes) .sendAsync(); }); return producer.flushAsync().whenComplete((ignored, cause) -> { if (null != callback) { callback.onCompletion(cause, ignored); } if (null == cause) { readToEnd(new CompletableFuture<>()); } }); } }
/** * This is called when client sends message to pulsar broker, before key and value gets serialized. * The method calls {@link ProducerInterceptor#beforeSend(Producer,Message)} method. Message returned from * first interceptor's beforeSend() is passed to the second interceptor beforeSend(), and so on in the * interceptor chain. The message returned from the last interceptor is returned from this method. * * This method does not throw exceptions. Exceptions thrown by any interceptor methods are caught and ignored. * If a interceptor in the middle of the chain, that normally modifies the message, throws an exception, * the next interceptor in the chain will be called with a message returned by the previous interceptor that did * not throw an exception. * * @param producer the producer which contains the interceptor. * @param message the message from client * @return the message to send to topic/partition */ public Message<T> beforeSend(Producer<T> producer, Message<T> message) { Message<T> interceptorMessage = message; for (int i = 0; i < interceptors.size(); i++) { try { interceptorMessage = interceptors.get(i).beforeSend(producer, interceptorMessage); } catch (Exception e) { if (message != null && producer != null) { log.warn("Error executing interceptor beforeSend callback for messageId: {}, topicName:{} ", message.getMessageId(), producer.getTopic(), e); } else { log.warn("Error Error executing interceptor beforeSend callback ", e); } } } return interceptorMessage; }
@Override public CompletableFuture<Void> sendAsync(Optional<String> key, byte[] payload) { TypedMessageBuilder<byte[]> msgBuilder = producer.newMessage().value(payload); if (key.isPresent()) { msgBuilder.key(key.get()); } return msgBuilder.sendAsync().thenApply(msgId -> null); }
@Override public void shutdown() { try { producer.close(); pulsarClient.close(); } catch (final Exception ex) { } }
producer.sendAsync(payloadData).thenRun(() -> { messagesSent.increment(); bytesSent.add(payloadData.length);
@Override public void writeFailed(Throwable th) { log.warn("[{}] Failed to send ack {}", producer.getTopic(), th.getMessage(), th); }
TypedMessageBuilder<byte[]> builder = producer.newMessage(); log.warn("[{}] Error occurred while producer handler was sending msg from {}: {}", producer.getTopic(), getRemote().getInetSocketAddress().toString(), exception.getMessage()); numMsgsFailed.increment();
@Override public void close() throws Exception { if (producer != null) { producer.close(); } // make sure we propagate pending errors checkErroneous(); }
@Override public void stop() { this.state = State.STOPPING; producer.closeAsync(); producer = null; this.state = State.STOPPED; }
@Override public void invoke(IN value, Context context) throws Exception { checkErroneous(); byte[] serializedValue = schema.serialize(value); MessageBuilder msgBuilder = MessageBuilder.create(); if (null != context.timestamp()) { msgBuilder = msgBuilder.setEventTime(context.timestamp()); } String msgKey = flinkPulsarKeyExtractor.getKey(value); if (null != msgKey) { msgBuilder = msgBuilder.setKey(msgKey); } Message message = msgBuilder .setContent(serializedValue) .build(); if (flushOnCheckpoint) { synchronized (pendingRecordsLock) { pendingRecords++; } } producer.sendAsync(message) .thenApply(successCallback) .exceptionally(failureCallback); }
@Override public void writeSuccess() { if (log.isDebugEnabled()) { log.debug("[{}] Ack was sent successfully to {}", producer.getTopic(), getRemote().getInetSocketAddress().toString()); } } });