@Override public void close() { if (receiver != null) { try { receiver.close().whenComplete((voidargs, error) -> { try { if (error != null) { logger.error("Exception during receiver close phase" + error.toString()); } ehClient.closeSync(); } catch (Exception e) { logger.error("Exception during ehclient close phase" + e.toString()); } }).get(); } catch (InterruptedException e) { logger.error("Exception occured during close phase" + e.toString()); } catch (ExecutionException e) { logger.error("Exception occured during close phase" + e.toString()); } logger.info("closed eventhub receiver: partitionId=" + partitionId); receiver = null; ehClient = null; } }
/** * This method is here to try and isolate the Azure related code as the PartitionReceiver cannot be mocked * with PowerMock due to it being final. Unfortunately it extends a base class and does not implement an interface * so even if we create a MockPartitionReciver, it will not work as the two classes are orthogonal. * * @param context - The processcontext for this processor * @param partitionId - The partition ID to retrieve a receiver by. * @return - Returns the events received from the EventBus. * @throws ProcessException -- If any exception is encountered, receiving events it is wrapped in a ProcessException * and then that exception is thrown. */ protected Iterable<EventData> receiveEvents(final ProcessContext context, final String partitionId) throws ProcessException { final PartitionReceiver receiver; try { receiver = getReceiver(context, partitionId); return receiver.receive(receiverFetchSize).get(); } catch (final IOException | ServiceBusException | ExecutionException | InterruptedException e) { throw new ProcessException(e); } }
PartitionReceiver getReceiver(final ProcessContext context, final String partitionId) throws IOException, ServiceBusException, ExecutionException, InterruptedException { PartitionReceiver existingReceiver = partitionToReceiverMap.get(partitionId); if (existingReceiver != null) { return existingReceiver; } // we want to avoid allowing multiple threads to create Receivers simultaneously because that could result in // having multiple Receivers for the same partition. So if the map does not contain a receiver for this partition, // we will enter a synchronized block and check again (because once we enter the synchronized block, we know that no // other thread is creating a client). If within the synchronized block, we still do not have an entry in the map, // it is up to use to create the receiver, initialize it, and then put it into the map. // We do not use the putIfAbsent method in order to do a CAS operation here because we want to also initialize the // receiver if and only if it is not present in the map. As a result, we need to initialize the receiver and add it // to the map atomically. Hence, the synchronized block. synchronized (this) { existingReceiver = partitionToReceiverMap.get(partitionId); if (existingReceiver != null) { return existingReceiver; } final String consumerGroupName = context.getProperty(CONSUMER_GROUP).getValue(); final PartitionReceiver receiver = eventHubClient.createReceiver( consumerGroupName, partitionId, configuredEnqueueTime == null ? Instant.now() : configuredEnqueueTime).get(); receiver.setReceiveTimeout(receiverFetchTimeout == null ? Duration.ofMillis(60000) : receiverFetchTimeout); partitionToReceiverMap.put(partitionId, receiver); return receiver; } }
private void renewPartitionReceiver(SystemStreamPartition ssp) { String streamId = config.getStreamId(ssp.getStream()); EventHubClientManager eventHubClientManager = perPartitionEventHubManagers.get(ssp); String offset = streamPartitionOffsets.get(ssp); Integer partitionId = ssp.getPartition().getPartitionId(); String consumerGroup = config.getStreamConsumerGroup(ssp.getSystem(), streamId); try { // Close current receiver streamPartitionReceivers.get(ssp).close().get(DEFAULT_SHUTDOWN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); // Recreate receiver PartitionReceiver receiver = eventHubClientManager.getEventHubClient() .createReceiverSync(consumerGroup, partitionId.toString(), EventPosition.fromOffset(offset, !offset.equals(EventHubSystemConsumer.START_OF_STREAM))); receiver.setPrefetchCount(prefetchCount); // Timeout for EventHubClient receive receiver.setReceiveTimeout(DEFAULT_EVENTHUB_RECEIVER_TIMEOUT); // Create and start receiver thread with handler receiver.setReceiveHandler(streamPartitionHandlers.get(ssp)); streamPartitionReceivers.put(ssp, receiver); } catch (Exception e) { eventHubNonTransientError.set(new SamzaException( String.format("Failed to recreate receiver for EventHubs after ReceiverHandlerError (ssp=%s)", ssp), e)); } }
receiver.setPrefetchCount(prefetchCount); receiver.setReceiveTimeout(DEFAULT_EVENTHUB_RECEIVER_TIMEOUT); receiver.setReceiveHandler(handler);
cleanupFuture = this.partitionReceiver.setReceiveHandler(null); } else { TRACE_LOGGER.debug(this.hostContext.withHostAndPartition(this.partitionContext, "partitionReceiver is null in cleanup")); .thenComposeAsync((partitionReceiverTemp) -> return (partitionReceiverTemp != null) ? partitionReceiverTemp.close() : CompletableFuture.completedFuture(null); }, this.hostContext.getExecutor()) .handleAsync((empty, e) ->
PowerMockito.when(mockPartitionReceiver.setReceiveHandler(any())).then((Answer<Void>) invocationOnMock -> { PartitionReceiveHandler handler = invocationOnMock.getArgumentAt(0, PartitionReceiveHandler.class); if (handler == null) {
@Override public EventDataWrap receive() { long start = System.currentTimeMillis(); Iterable<EventData> receivedEvents = null; /*Get one message at a time for backward compatibility behaviour*/ try { receivedEvents = receiver.receiveSync(1); } catch (ServiceBusException e) { logger.error("Exception occured during receive" + e.toString()); return null; } long end = System.currentTimeMillis(); long millis = (end - start); receiveApiLatencyMean.update(millis); receiveApiCallCount.incr(); if (receivedEvents == null || receivedEvents.spliterator().getExactSizeIfKnown() == 0) { return null; } receiveMessageCount.incr(); EventData receivedEvent = receivedEvents.iterator().next(); MessageId messageId = new MessageId(partitionId, receivedEvent.getSystemProperties().getOffset(), receivedEvent.getSystemProperties().getSequenceNumber()); return EventDataWrap.create(receivedEvent, messageId); }
@Override public void onReceive(Iterable<EventData> events) { if (this.hostContext.getEventProcessorOptions().getReceiverRuntimeMetricEnabled()) { this.partitionContext.setRuntimeInformation(this.partitionReceiver.getRuntimeInformation());
this.partitionReceiver.setReceiveHandler(this, this.hostContext.getEventProcessorOptions().getInvokeProcessorAfterReceiveTimeout()); } else { Exception notifyWith = (Exception) LoggingUtils.unwrapException(e, null);
private void receiveMessages(PartitionReceiver receiver, int numMessages) throws EventHubException { int count = 0; while (count < numMessages) { Iterable<EventData> messages = receiver.receiveSync(100); if (messages == null) { break; } for (EventData data : messages) { count++; LOG.info("Data" + new String(data.getBytes())); } } }
/** * Synchronous version of {@link #receive}. * * @param maxEventCount maximum number of {@link EventData}'s that this call should return * @return Batch of {@link EventData}'s from the partition on which this receiver is created. Returns 'null' if no {@link EventData} is present. * @throws EventHubException if ServiceBus client encountered any unrecoverable/non-transient problems during {@link #receive} */ default Iterable<EventData> receiveSync(final int maxEventCount) throws EventHubException { return ExceptionUtil.sync(() -> this.receive(maxEventCount).get()); }
@OnStopped public void tearDown() throws ProcessException { for (final PartitionReceiver receiver : partitionToReceiverMap.values()) { if (null != receiver) { receiver.close(); } } partitionToReceiverMap.clear(); try { if (null != eventHubClient) { eventHubClient.closeSync(); } } catch (final ServiceBusException e) { throw new ProcessException(e); } }
this.partitionReceiver.setReceiveTimeout(this.hostContext.getEventProcessorOptions().getReceiveTimeOut());
@Override public void accept(PartitionReceiver receiver) { while (true) { try { Iterable<EventData> receivedEvents = receiver.receive(MAX_MESSAGES).get(); int batchSize = 0; if (receivedEvents != null) { for (EventData receivedEvent : receivedEvents) { String deviceId = (String) receivedEvent.getProperties().get("iothub-connection-device-id"); Map<String, Object> deviceState = null; try { deviceState = ParaObjectUtils.getJsonReader(Map.class).readValue(receivedEvent.getBytes()); logger.debug("Message received from Azure: {}", deviceState); } catch (Exception e) { } if (deviceState != null) { Thing t = thingFromCloudID(deviceId); if (t != null) { t.setDeviceState(deviceState); Para.getDAO().update(t.getAppid(), t); } } batchSize++; } } logger.debug("Received {} messages from Azure for partition {}.", batchSize, partitionId); } catch (Exception e) { logger.warn("Failed to receive messages: {}", e.getMessage()); } } }
@Override public void accept(PartitionReceiver receiver) { while (true) { try { Iterable<EventData> receivedEvents = receiver.receive(MAX_MESSAGES).get(); int batchSize = 0; if (receivedEvents != null) { for (EventData receivedEvent : receivedEvents) { String deviceId = (String) receivedEvent.getProperties().get("iothub-connection-device-id"); Map<String, Object> deviceState = null; try { deviceState = ParaObjectUtils.getJsonReader(Map.class).readValue(receivedEvent.getBytes()); logger.debug("Message received from Azure: {}", deviceState); } catch (Exception e) { } if (deviceState != null) { Thing t = thingFromCloudID(deviceId); if (t != null) { t.setDeviceState(deviceState); Para.getDAO().update(t.getAppid(), t); } } batchSize++; } } logger.debug("Received {} messages from Azure for partition {}.", batchSize, partitionId); } catch (Exception e) { logger.warn("Failed to receive messages: {}", e.getMessage()); } } }