private static EventProcessorOptions buildEventProcessorOptions(StartPosition startPosition) { EventProcessorOptions options = EventProcessorOptions.getDefaultOptions(); if (startPosition == StartPosition.EARLIEST) { options.setInitialPositionProvider((s) -> EventPosition.fromStartOfStream()); } else /* StartPosition.LATEST */ { options.setInitialPositionProvider((s) -> EventPosition.fromEndOfStream()); } return options; }
private static EventHubClient receiveEventsAsync(final String partitionId) { EventHubClient client = null; try { client = EventHubClient.createSync(EVENTHUB_CONN_STR, Para.getExecutorService()); client.createReceiver(EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, partitionId, EventPosition.fromEnqueuedTime(Instant.now())). thenAccept(new Receiver(partitionId)); } catch (Exception e) { logger.warn("Couldn't start receiving messages from Azure cloud: {}", e.getMessage()); } return client; }
@Override public EventPosition apply(String t) { return EventPosition.fromStartOfStream(); } }
.createReceiverSync(consumerGroup, partitionId.toString(), EventPosition.fromEnqueuedTime(Instant.now())); } else { EventPosition.fromOffset(offset, /* inclusiveFlag */false));
@Override public EventPosition apply(String t) { return EventPosition.fromEndOfStream(); } }
EventPosition checkpointToOffset(Checkpoint startingCheckpoint) { EventPosition startAt = null; if (startingCheckpoint == null) { // No checkpoint was ever stored. Use the initialOffsetProvider instead. Function<String, EventPosition> initialPositionProvider = this.hostContext.getEventProcessorOptions().getInitialPositionProvider(); TRACE_LOGGER.debug(this.hostContext.withHostAndPartition(this.partitionId, "Calling user-provided initial position provider")); startAt = initialPositionProvider.apply(this.partitionId); // Leave this.offset as null. The initialPositionProvider cannot provide enough information to write a valid checkpoint: // at most if will give one of offset or sequence number, and if it is a starting time then it doesn't have either. TRACE_LOGGER.info(this.hostContext.withHostAndPartition(this.partitionId, "Initial position provided: " + startAt)); } else { // Checkpoint is valid, use it. this.offset = startingCheckpoint.getOffset(); startAt = EventPosition.fromOffset(this.offset); this.sequenceNumber = startingCheckpoint.getSequenceNumber(); TRACE_LOGGER.info(this.hostContext.withHostAndPartition(this.partitionId, "Retrieved starting offset " + this.offset + "//" + this.sequenceNumber)); } return startAt; }
static LinkedList<EventData> toEventDataCollection(final Collection<Message> messages, final PassByRef<MessageWrapper> lastMessageRef) { if (messages == null) { return null; } LinkedList<EventData> events = new LinkedList<>(); for (Message message : messages) { EventData eventData = new EventDataImpl(message); events.add(eventData); if (lastMessageRef != null) { lastMessageRef.set(new MessageWrapper(message, EventPosition.fromSequenceNumber(eventData.getSystemProperties().getSequenceNumber(), true))); } } return events; }
private PartitionReceiverImpl(MessagingFactory factory, final String eventHubName, final String consumerGroupName, final String partitionId, final EventPositionImpl eventPosition, final Long epoch, final boolean isEpochReceiver, final ReceiverOptions receiverOptions, final ScheduledExecutorService executor) { super("PartitionReceiverImpl".concat(StringUtil.getRandomString()), null, executor); this.underlyingFactory = factory; this.eventHubName = eventHubName; this.consumerGroupName = consumerGroupName; this.partitionId = partitionId; this.eventPosition = eventPosition; this.epoch = epoch; this.isEpochReceiver = isEpochReceiver; this.receiveHandlerLock = new Object(); this.receiverOptions = receiverOptions; this.runtimeInformation = (this.receiverOptions != null && this.receiverOptions.getReceiverRuntimeMetricEnabled()) ? new ReceiverRuntimeInformation(partitionId) : null; this.currentEventPosition = EventPosition.fromStartOfStream(); }
.then((Answer<PartitionReceiver>) invocationOnMock -> { String partitionId = invocationOnMock.getArgumentAt(1, String.class); startingOffsets.put(partitionId, EventPosition.fromEndOfStream()); return mockPartitionReceiver; });
private void renewPartitionReceiver(SystemStreamPartition ssp) { String streamId = config.getStreamId(ssp.getStream()); EventHubClientManager eventHubClientManager = perPartitionEventHubManagers.get(ssp); String offset = streamPartitionOffsets.get(ssp); Integer partitionId = ssp.getPartition().getPartitionId(); String consumerGroup = config.getStreamConsumerGroup(ssp.getSystem(), streamId); try { // Close current receiver streamPartitionReceivers.get(ssp).close().get(DEFAULT_SHUTDOWN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); // Recreate receiver PartitionReceiver receiver = eventHubClientManager.getEventHubClient() .createReceiverSync(consumerGroup, partitionId.toString(), EventPosition.fromOffset(offset, !offset.equals(EventHubSystemConsumer.START_OF_STREAM))); receiver.setPrefetchCount(prefetchCount); // Timeout for EventHubClient receive receiver.setReceiveTimeout(DEFAULT_EVENTHUB_RECEIVER_TIMEOUT); // Create and start receiver thread with handler receiver.setReceiveHandler(streamPartitionHandlers.get(ssp)); streamPartitionReceivers.put(ssp, receiver); } catch (Exception e) { eventHubNonTransientError.set(new SamzaException( String.format("Failed to recreate receiver for EventHubs after ReceiverHandlerError (ssp=%s)", ssp), e)); } }
private static EventProcessorOptions buildEventProcessorOptions(StartPosition startPosition) { EventProcessorOptions options = EventProcessorOptions.getDefaultOptions(); if (startPosition == StartPosition.EARLIEST) { options.setInitialPositionProvider((s) -> EventPosition.fromStartOfStream()); } else /* StartPosition.LATEST */ { options.setInitialPositionProvider((s) -> EventPosition.fromEndOfStream()); } return options; }
@Test public void testReceive() throws EventHubException { EventHubClientManagerFactory clientFactory = new EventHubClientManagerFactory(); EventHubClientManager wrapper = clientFactory .getEventHubClientManager(SYSTEM_NAME, STREAM_NAME1, new EventHubConfig(createEventHubConfig())); wrapper.init(); EventHubClient client = wrapper.getEventHubClient(); PartitionReceiver receiver = client.createReceiverSync(EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, "0", EventPosition.fromStartOfStream()); receiveMessages(receiver, 300); }
consumer.start(); Assert.assertEquals(EventPosition.fromOffset(EventHubSystemConsumer.START_OF_STREAM, false).toString(), eventHubClientWrapperFactory.getPartitionOffset(String.valueOf(partitionId)).toString());
private static EventHubClient receiveEventsAsync(final String partitionId) { EventHubClient client = null; try { client = EventHubClient.createSync(EVENTHUB_CONN_STR, Para.getExecutorService()); client.createReceiver(EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, partitionId, EventPosition.fromEnqueuedTime(Instant.now())). thenAccept(new Receiver(partitionId)); } catch (Exception e) { logger.warn("Couldn't start receiving messages from Azure cloud: {}", e.getMessage()); } return client; }