private void putEventHubAttributes(Map<String, String> attributes, String eventHubName, String partitionId, EventData eventData) { final EventData.SystemProperties systemProperties = eventData.getSystemProperties(); if (null != systemProperties) { attributes.put("eventhub.enqueued.timestamp", String.valueOf(systemProperties.getEnqueuedTime())); attributes.put("eventhub.offset", systemProperties.getOffset()); attributes.put("eventhub.sequence", String.valueOf(systemProperties.getSequenceNumber())); } attributes.put("eventhub.name", eventHubName); attributes.put("eventhub.partition", partitionId); }
@Override public EventDataWrap receive() { EventDataWrap eventDatawrap = receiver.receive(); if (eventDatawrap != null) { lastOffset = eventDatawrap.getEventData().getSystemProperties().getOffset(); } return eventDatawrap; }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); byte[] messageData = null; if (eventData.getBytes() != null) { messageData = eventData.getBytes(); } else if (eventData.getObject() != null) { try { messageData = SerializeDeserializeUtil.serialize(eventData.getObject()); } catch (IOException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw new RuntimeException(e); } } Map<String, Object> metaDataMap = eventData.getProperties(); Map<String, Object> systemMetaDataMap = eventData.getSystemProperties(); fieldContents.add(messageData); fieldContents.add(metaDataMap); fieldContents.add(systemMetaDataMap); return fieldContents; }
@Override public EventDataWrap receive() { long start = System.currentTimeMillis(); Iterable<EventData> receivedEvents = null; /*Get one message at a time for backward compatibility behaviour*/ try { receivedEvents = receiver.receiveSync(1); } catch (ServiceBusException e) { logger.error("Exception occured during receive" + e.toString()); return null; } long end = System.currentTimeMillis(); long millis = (end - start); receiveApiLatencyMean.update(millis); receiveApiCallCount.incr(); if (receivedEvents == null || receivedEvents.spliterator().getExactSizeIfKnown() == 0) { return null; } receiveMessageCount.incr(); EventData receivedEvent = receivedEvents.iterator().next(); MessageId messageId = new MessageId(partitionId, receivedEvent.getSystemProperties().getOffset(), receivedEvent.getSystemProperties().getSequenceNumber()); return EventDataWrap.create(receivedEvent, messageId); }
final EventData.SystemProperties systemProperties = eventData.getSystemProperties();
public static String toString(EventData eventData) { Map<String, Object> map = new LinkedHashMap<>(); map.put("body", eventData.getObject()); map.put("offset", eventData.getSystemProperties().getOffset()); map.put("sequenceNumber", eventData.getSystemProperties().getSequenceNumber()); map.put("enqueuedTime", eventData.getSystemProperties().getEnqueuedTime()); return map.toString(); } }
public static String toString(EventData eventData) { Map<String, Object> map = new LinkedHashMap<>(); map.put("body", eventData.getObject()); map.put("offset", eventData.getSystemProperties().getOffset()); map.put("sequenceNumber", eventData.getSystemProperties().getSequenceNumber()); map.put("enqueuedTime", eventData.getSystemProperties().getEnqueuedTime()); return map.toString(); } }
void setOffsetAndSequenceNumber(EventData event) { if (event.getSystemProperties().getSequenceNumber() >= this.sequenceNumber) { this.offset = event.getSystemProperties().getOffset(); this.sequenceNumber = event.getSystemProperties().getSequenceNumber(); } else { TRACE_LOGGER.info(this.hostContext.withHostAndPartition(this.partitionId, "setOffsetAndSequenceNumber(" + event.getSystemProperties().getOffset() + "//" + event.getSystemProperties().getSequenceNumber() + ") would move backwards, ignoring")); } }
/** * Writes the position of the provided EventData instance to the checkpoint store via the checkpoint manager. * <p> * It is important to check the result in order to detect failures. * * @param event A received EventData * @return CompletableFuture {@literal ->} null when the checkpoint has been persisted successfully, completes exceptionally on error. */ public CompletableFuture<Void> checkpoint(EventData event) { CompletableFuture<Void> result = null; if (event == null) { result = new CompletableFuture<Void>(); result.completeExceptionally(new IllegalArgumentException("Cannot checkpoint with null EventData")); } else { result = checkpoint(new Checkpoint(this.partitionId, event.getSystemProperties().getOffset(), event.getSystemProperties().getSequenceNumber())); } return result; }
@Override public int compareTo(EventData other) { return Long.compare( this.getSystemProperties().getSequenceNumber(), other.getSystemProperties().getSequenceNumber() ); } }
/** * If what we've read puts us at or over our count since last checkpoint, checkpoint and return * true. */ boolean maybeCheckpoint(PartitionContext context, EventData data, int spansRead) throws InterruptedException, ExecutionException { if (!shouldCheckPoint(spansRead)) return false; if (logger.isLoggable(Level.FINE)) { logger.log( Level.FINE, "Partition " + partitionId(context) + " checkpointing at " + data.getSystemProperties().getOffset() + "," + data.getSystemProperties().getSequenceNumber()); } checkpoint(context, data); return true; }
@Override public void onEvents(PartitionContext context, Iterable<EventData> events) throws Exception { LOGGER.info("Partition " + context.getPartitionId() + " got event batch"); int eventCount = 0; for (EventData data : events) { try { LOGGER.info("SAMPLE (" + context.getPartitionId() + "," + data.getSystemProperties().getOffset() + "," + data.getSystemProperties().getSequenceNumber() + "): " + new String(data.getBytes(), "UTF8")); eventCount++; this.checkpointBatchingCount++; if ((checkpointBatchingCount % 5) == 0) { LOGGER.info("SAMPLE: Partition " + context.getPartitionId() + " checkpointing at " + data.getSystemProperties().getOffset() + "," + data.getSystemProperties().getSequenceNumber()); context.checkpoint(data).get(); } } catch (Exception e) { LOGGER.error("Processing failed for an event: " + e.toString(), e); } } LOGGER.info("Partition " + context.getPartitionId() + " batch size was " + eventCount + " for host " + context.getOwner()); } }
@Override public void onReceive(Iterable<EventData> events) { if (events != null) { events.forEach(event -> { byte[] eventDataBody = event.getBytes(); if (interceptor != null) { eventDataBody = interceptor.intercept(eventDataBody); } String offset = event.getSystemProperties().getOffset(); Object partitionKey = event.getSystemProperties().getPartitionKey(); if (partitionKey == null) { partitionKey = event.getProperties().get(EventHubSystemProducer.KEY); } try { updateMetrics(event); // note that the partition key can be null put(ssp, new EventHubIncomingMessageEnvelope(ssp, offset, partitionKey, eventDataBody, event)); } catch (InterruptedException e) { String msg = String.format("Interrupted while adding the event from ssp %s to dispatch queue.", ssp); LOG.error(msg, e); throw new SamzaException(msg, e); } // Cache latest checkpoint streamPartitionOffsets.put(ssp, offset); }); } }
static LinkedList<EventData> toEventDataCollection(final Collection<Message> messages, final PassByRef<MessageWrapper> lastMessageRef) { if (messages == null) { return null; } LinkedList<EventData> events = new LinkedList<>(); for (Message message : messages) { EventData eventData = new EventDataImpl(message); events.add(eventData); if (lastMessageRef != null) { lastMessageRef.set(new MessageWrapper(message, EventPosition.fromSequenceNumber(eventData.getSystemProperties().getSequenceNumber(), true))); } } return events; }
private void updateMetrics(EventData event) { int eventDataLength = event.getBytes() == null ? 0 : event.getBytes().length; eventReadRate.inc(); aggEventReadRate.inc(); eventByteReadRate.inc(eventDataLength); aggEventByteReadRate.inc(eventDataLength); long latencyMs = Duration.between(event.getSystemProperties().getEnqueuedTime(), Instant.now()).toMillis(); readLatency.update(latencyMs); aggConsumptionLagMs.update(latencyMs); }
private void verifyEvents(List<IncomingMessageEnvelope> messages, List<EventData> eventDataList, Interceptor interceptor) { Assert.assertEquals(messages.size(), eventDataList.size()); for (int i = 0; i < messages.size(); i++) { IncomingMessageEnvelope message = messages.get(i); EventData eventData = eventDataList.get(i); Assert.assertEquals(message.getKey(), eventData.getSystemProperties().getPartitionKey()); Assert.assertEquals(message.getMessage(), interceptor.intercept(eventData.getBytes())); Assert.assertEquals(message.getOffset(), eventData.getSystemProperties().getOffset()); } }
@Test public void testSinglePartitionConsumptionHappyPath() throws Exception { int partitionId = 0; TestMetricsRegistry testMetrics = new TestMetricsRegistry(); SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, STREAM_NAME1, new Partition(partitionId)); Config eventHubConfig = createEventHubConfig(); EventHubSystemFactory factory = new EventHubSystemFactory(); SystemConsumer consumer = factory.getConsumer(SYSTEM_NAME, eventHubConfig, testMetrics); consumer.register(ssp, EventHubSystemConsumer.START_OF_STREAM); consumer.start(); int numEvents = 0; int numRetries = 20; while (numRetries-- > 0) { List<IncomingMessageEnvelope> result = consumer.poll(Collections.singleton(ssp), 2000).get(ssp); numEvents = result == null ? 0 : result.size(); if (numEvents > 0) { EventHubIncomingMessageEnvelope eventData = (EventHubIncomingMessageEnvelope) result.get(0); System.out.println("System properties: " + eventData.getEventData().getSystemProperties()); System.out.println("Key: " + new String((byte[]) eventData.getKey())); System.out.println("Message: " + new String((byte[]) eventData.getMessage())); } System.out.println("Retries left: " + numRetries); } Assert.assertTrue(numEvents > 0); }