private void putEventHubAttributes(Map<String, String> attributes, String eventHubName, String partitionId, EventData eventData) { final EventData.SystemProperties systemProperties = eventData.getSystemProperties(); if (null != systemProperties) { attributes.put("eventhub.enqueued.timestamp", String.valueOf(systemProperties.getEnqueuedTime())); attributes.put("eventhub.offset", systemProperties.getOffset()); attributes.put("eventhub.sequence", String.valueOf(systemProperties.getSequenceNumber())); } attributes.put("eventhub.name", eventHubName); attributes.put("eventhub.partition", partitionId); }
@Override public EventDataWrap receive() { EventDataWrap eventDatawrap = receiver.receive(); if (eventDatawrap != null) { lastOffset = eventDatawrap.getEventData().getSystemProperties().getOffset(); } return eventDatawrap; }
protected void sendMessage(final byte[] buffer) throws ProcessException { final EventHubClient sender = senderQueue.poll(); if(null != sender) { try { sender.sendSync(new EventData(buffer)); } catch (final ServiceBusException sbe) { throw new ProcessException("Caught exception trying to send message to eventbus", sbe); } finally { senderQueue.offer(sender); } }else{ throw new ProcessException("No EventHubClients are configured for sending"); } } }
@Override public EventDataWrap receive() { long start = System.currentTimeMillis(); Iterable<EventData> receivedEvents = null; /*Get one message at a time for backward compatibility behaviour*/ try { receivedEvents = receiver.receiveSync(1); } catch (ServiceBusException e) { logger.error("Exception occured during receive" + e.toString()); return null; } long end = System.currentTimeMillis(); long millis = (end - start); receiveApiLatencyMean.update(millis); receiveApiCallCount.incr(); if (receivedEvents == null || receivedEvents.spliterator().getExactSizeIfKnown() == 0) { return null; } receiveMessageCount.incr(); EventData receivedEvent = receivedEvents.iterator().next(); MessageId messageId = new MessageId(partitionId, receivedEvent.getSystemProperties().getOffset(), receivedEvent.getSystemProperties().getSequenceNumber()); return EventDataWrap.create(receivedEvent, messageId); }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); byte[] messageData = null; if (eventData.getBytes() != null) { messageData = eventData.getBytes(); } else if (eventData.getObject() != null) { try { messageData = SerializeDeserializeUtil.serialize(eventData.getObject()); } catch (IOException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw new RuntimeException(e); } } Map<String, Object> metaDataMap = eventData.getProperties(); Map<String, Object> systemMetaDataMap = eventData.getSystemProperties(); fieldContents.add(messageData); fieldContents.add(metaDataMap); fieldContents.add(systemMetaDataMap); return fieldContents; }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); String messageData = ""; if (eventData.getBytes() != null) { messageData = new String(eventData.getBytes()); } /*Will only serialize AMQPValue type*/ else if (eventData.getObject() != null) { try { if (!(eventData.getObject() instanceof List)) { messageData = eventData.getObject().toString(); } else { throw new RuntimeException("Cannot serialize the given AMQP type"); } } catch (RuntimeException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw e; } } Map<String, Object> metaDataMap = eventData.getProperties(); fieldContents.add(messageData); fieldContents.add(metaDataMap); return fieldContents; }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); String messageData = ""; if (eventData.getBytes() != null) { messageData = new String(eventData.getBytes()); } /*Will only serialize AMQPValue type*/ else if (eventData.getObject() != null) { try { if (!(eventData.getObject() instanceof List)) { messageData = eventData.getObject().toString(); } else { throw new RuntimeException("Cannot serialize the given AMQP type."); } } catch (RuntimeException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw e; } } fieldContents.add(messageData); return fieldContents; }
@OnStopped public void tearDown() throws ProcessException { for (final PartitionReceiver receiver : partitionToReceiverMap.values()) { if (null != receiver) { receiver.close(); } } partitionToReceiverMap.clear(); try { if (null != eventHubClient) { eventHubClient.closeSync(); } } catch (final ServiceBusException e) { throw new ProcessException(e); } }
private void writeFlowFiles(PartitionContext context, Iterable<EventData> messages, ProcessSession session, StopWatch stopWatch) { final String eventHubName = context.getEventHubPath(); final String partitionId = context.getPartitionId(); final String consumerGroup = context.getConsumerGroupName(); messages.forEach(eventData -> { FlowFile flowFile = session.create(); final Map<String, String> attributes = new HashMap<>(); putEventHubAttributes(attributes, eventHubName, partitionId, eventData); flowFile = session.putAllAttributes(flowFile, attributes); flowFile = session.write(flowFile, out -> { out.write(eventData.getBytes()); }); transferTo(REL_SUCCESS, session, stopWatch, eventHubName, partitionId, consumerGroup, flowFile); }); }
@Override public void execute(Tuple tuple) { try { EventData sendEvent = new EventData(boltConfig.getEventDataFormat().serialize(tuple)); if (boltConfig.getPartitionMode() && sender != null) { sender.sendSync(sendEvent); } else if (boltConfig.getPartitionMode() && sender == null) { throw new EventHubException("Sender is null"); } else if (!boltConfig.getPartitionMode() && ehClient != null) { ehClient.sendSync(sendEvent); } else if (!boltConfig.getPartitionMode() && ehClient == null) { throw new EventHubException("ehclient is null"); } collector.ack(tuple); } catch (EventHubException ex) { collector.reportError(ex); collector.fail(tuple); } catch (ServiceBusException e) { collector.reportError(e); collector.fail(tuple); } }
@Override public void cleanup() { if (sender != null) { try { sender.close().whenComplete((voidargs, error) -> { try { if (error != null) { logger.error("Exception during sender cleanup phase" + error.toString()); } ehClient.closeSync(); } catch (Exception e) { logger.error("Exception during ehclient cleanup phase" + e.toString()); } }).get(); } catch (InterruptedException e) { logger.error("Exception occured during cleanup phase" + e.toString()); } catch (ExecutionException e) { logger.error("Exception occured during cleanup phase" + e.toString()); } logger.info("Eventhub Bolt cleaned up"); sender = null; ehClient = null; } }
PartitionReceiver getReceiver(final ProcessContext context, final String partitionId) throws IOException, ServiceBusException, ExecutionException, InterruptedException { PartitionReceiver existingReceiver = partitionToReceiverMap.get(partitionId); if (existingReceiver != null) { return existingReceiver; } // we want to avoid allowing multiple threads to create Receivers simultaneously because that could result in // having multiple Receivers for the same partition. So if the map does not contain a receiver for this partition, // we will enter a synchronized block and check again (because once we enter the synchronized block, we know that no // other thread is creating a client). If within the synchronized block, we still do not have an entry in the map, // it is up to use to create the receiver, initialize it, and then put it into the map. // We do not use the putIfAbsent method in order to do a CAS operation here because we want to also initialize the // receiver if and only if it is not present in the map. As a result, we need to initialize the receiver and add it // to the map atomically. Hence, the synchronized block. synchronized (this) { existingReceiver = partitionToReceiverMap.get(partitionId); if (existingReceiver != null) { return existingReceiver; } final String consumerGroupName = context.getProperty(CONSUMER_GROUP).getValue(); final PartitionReceiver receiver = eventHubClient.createReceiver( consumerGroupName, partitionId, configuredEnqueueTime == null ? Instant.now() : configuredEnqueueTime).get(); receiver.setReceiveTimeout(receiverFetchTimeout == null ? Duration.ofMillis(60000) : receiverFetchTimeout); partitionToReceiverMap.put(partitionId, receiver); return receiver; } }
@Override public void prepare(Map<String, Object> config, TopologyContext context, OutputCollector collector) { this.collector = collector; String myPartitionId = null; if (boltConfig.getPartitionMode()) { // We can use the task index (starting from 0) as the partition ID myPartitionId = "" + context.getThisTaskIndex(); } logger.info("creating sender: " + boltConfig.getConnectionString() + ", " + boltConfig.getEntityPath() + ", " + myPartitionId); try { ehClient = EventHubClient.createFromConnectionStringSync(boltConfig.getConnectionString()); if (boltConfig.getPartitionMode()) { sender = ehClient.createPartitionSenderSync(Integer.toString(context.getThisTaskIndex())); } } catch (Exception ex) { collector.reportError(ex); throw new RuntimeException(ex); } }
@OnStopped public void tearDown() { EventHubClient sender; while ((sender = senderQueue.poll()) != null) { sender.close(); } }
/** * This method is here to try and isolate the Azure related code as the PartitionReceiver cannot be mocked * with PowerMock due to it being final. Unfortunately it extends a base class and does not implement an interface * so even if we create a MockPartitionReciver, it will not work as the two classes are orthogonal. * * @param context - The processcontext for this processor * @param partitionId - The partition ID to retrieve a receiver by. * @return - Returns the events received from the EventBus. * @throws ProcessException -- If any exception is encountered, receiving events it is wrapped in a ProcessException * and then that exception is thrown. */ protected Iterable<EventData> receiveEvents(final ProcessContext context, final String partitionId) throws ProcessException { final PartitionReceiver receiver; try { receiver = getReceiver(context, partitionId); return receiver.receive(receiverFetchSize).get(); } catch (final IOException | ServiceBusException | ExecutionException | InterruptedException e) { throw new ProcessException(e); } }
long start = System.currentTimeMillis(); try { ehClient = EventHubClient.createFromConnectionStringSync(connectionString); receiver = ehClient.createEpochReceiverSync( consumerGroupName, partitionId, 1); } else if (filter.getTime() != null) { receiver = ehClient.createEpochReceiverSync( consumerGroupName, partitionId,
protected void setupReceiver(final String connectionString) throws ProcessException { try { eventHubClient = EventHubClient.createFromConnectionString(connectionString).get(); } catch (InterruptedException | ExecutionException | IOException | ServiceBusException e) { throw new ProcessException(e); } }
final EventData.SystemProperties systemProperties = eventData.getSystemProperties(); attributes.put("eventhub.enqueued.timestamp", String.valueOf(systemProperties.getEnqueuedTime())); attributes.put("eventhub.offset", systemProperties.getOffset()); attributes.put("eventhub.sequence", String.valueOf(systemProperties.getSequenceNumber())); out.write(eventData.getBytes()); });
@Override public void close() { if (receiver != null) { try { receiver.close().whenComplete((voidargs, error) -> { try { if (error != null) { logger.error("Exception during receiver close phase" + error.toString()); } ehClient.closeSync(); } catch (Exception e) { logger.error("Exception during ehclient close phase" + e.toString()); } }).get(); } catch (InterruptedException e) { logger.error("Exception occured during close phase" + e.toString()); } catch (ExecutionException e) { logger.error("Exception occured during close phase" + e.toString()); } logger.info("closed eventhub receiver: partitionId=" + partitionId); receiver = null; ehClient = null; } }
protected EventHubClient createEventHubClient(final String namespace, final String eventHubName, final String policyName, final String policyKey) throws ProcessException{ try { return EventHubClient.createFromConnectionString(getConnectionString(namespace, eventHubName, policyName, policyKey)).get(); } catch (InterruptedException | ExecutionException | IOException | ServiceBusException | IllegalConnectionStringFormatException e) { getLogger().error("Failed to create EventHubClient due to {}", e); throw new ProcessException(e); } } protected String getConnectionString(final String namespace, final String eventHubName, final String policyName, final String policyKey){