@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); byte[] messageData = null; if (eventData.getBytes() != null) { messageData = eventData.getBytes(); } else if (eventData.getObject() != null) { try { messageData = SerializeDeserializeUtil.serialize(eventData.getObject()); } catch (IOException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw new RuntimeException(e); } } Map<String, Object> metaDataMap = eventData.getProperties(); Map<String, Object> systemMetaDataMap = eventData.getSystemProperties(); fieldContents.add(messageData); fieldContents.add(metaDataMap); fieldContents.add(systemMetaDataMap); return fieldContents; }
protected void sendMessage(final byte[] buffer) throws ProcessException { final EventHubClient sender = senderQueue.poll(); if(null != sender) { try { sender.sendSync(new EventData(buffer)); } catch (final ServiceBusException sbe) { throw new ProcessException("Caught exception trying to send message to eventbus", sbe); } finally { senderQueue.offer(sender); } }else{ throw new ProcessException("No EventHubClients are configured for sending"); } } }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); String messageData = ""; if (eventData.getBytes() != null) { messageData = new String(eventData.getBytes()); } /*Will only serialize AMQPValue type*/ else if (eventData.getObject() != null) { try { if (!(eventData.getObject() instanceof List)) { messageData = eventData.getObject().toString(); } else { throw new RuntimeException("Cannot serialize the given AMQP type"); } } catch (RuntimeException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw e; } } Map<String, Object> metaDataMap = eventData.getProperties(); fieldContents.add(messageData); fieldContents.add(metaDataMap); return fieldContents; }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); String messageData = ""; if (eventData.getBytes() != null) { messageData = new String(eventData.getBytes()); } /*Will only serialize AMQPValue type*/ else if (eventData.getObject() != null) { try { if (!(eventData.getObject() instanceof List)) { messageData = eventData.getObject().toString(); } else { throw new RuntimeException("Cannot serialize the given AMQP type."); } } catch (RuntimeException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw e; } } fieldContents.add(messageData); return fieldContents; }
final EventData.SystemProperties systemProperties = eventData.getSystemProperties(); out.write(eventData.getBytes()); });
private void writeFlowFiles(PartitionContext context, Iterable<EventData> messages, ProcessSession session, StopWatch stopWatch) { final String eventHubName = context.getEventHubPath(); final String partitionId = context.getPartitionId(); final String consumerGroup = context.getConsumerGroupName(); messages.forEach(eventData -> { FlowFile flowFile = session.create(); final Map<String, String> attributes = new HashMap<>(); putEventHubAttributes(attributes, eventHubName, partitionId, eventData); flowFile = session.putAllAttributes(flowFile, attributes); flowFile = session.write(flowFile, out -> { out.write(eventData.getBytes()); }); transferTo(REL_SUCCESS, session, stopWatch, eventHubName, partitionId, consumerGroup, flowFile); }); }
private void putEventHubAttributes(Map<String, String> attributes, String eventHubName, String partitionId, EventData eventData) { final EventData.SystemProperties systemProperties = eventData.getSystemProperties(); if (null != systemProperties) { attributes.put("eventhub.enqueued.timestamp", String.valueOf(systemProperties.getEnqueuedTime())); attributes.put("eventhub.offset", systemProperties.getOffset()); attributes.put("eventhub.sequence", String.valueOf(systemProperties.getSequenceNumber())); } attributes.put("eventhub.name", eventHubName); attributes.put("eventhub.partition", partitionId); }
@Override public void onReceive(Iterable<EventData> events) { if (events != null) { events.forEach(event -> { byte[] eventDataBody = event.getBytes(); if (interceptor != null) { eventDataBody = interceptor.intercept(eventDataBody); } String offset = event.getSystemProperties().getOffset(); Object partitionKey = event.getSystemProperties().getPartitionKey(); if (partitionKey == null) { partitionKey = event.getProperties().get(EventHubSystemProducer.KEY); } try { updateMetrics(event); // note that the partition key can be null put(ssp, new EventHubIncomingMessageEnvelope(ssp, offset, partitionKey, eventDataBody, event)); } catch (InterruptedException e) { String msg = String.format("Interrupted while adding the event from ssp %s to dispatch queue.", ssp); LOG.error(msg, e); throw new SamzaException(msg, e); } // Cache latest checkpoint streamPartitionOffsets.put(ssp, offset); }); } }
public static String toString(EventData eventData) { Map<String, Object> map = new LinkedHashMap<>(); map.put("body", eventData.getObject()); map.put("offset", eventData.getSystemProperties().getOffset()); map.put("sequenceNumber", eventData.getSystemProperties().getSequenceNumber()); map.put("enqueuedTime", eventData.getSystemProperties().getEnqueuedTime()); return map.toString(); } }
@Override public void accept(PartitionReceiver receiver) { while (true) { try { Iterable<EventData> receivedEvents = receiver.receive(MAX_MESSAGES).get(); int batchSize = 0; if (receivedEvents != null) { for (EventData receivedEvent : receivedEvents) { String deviceId = (String) receivedEvent.getProperties().get("iothub-connection-device-id"); Map<String, Object> deviceState = null; try { deviceState = ParaObjectUtils.getJsonReader(Map.class).readValue(receivedEvent.getBytes()); logger.debug("Message received from Azure: {}", deviceState); } catch (Exception e) { } if (deviceState != null) { Thing t = thingFromCloudID(deviceId); if (t != null) { t.setDeviceState(deviceState); Para.getDAO().update(t.getAppid(), t); } } batchSize++; } } logger.debug("Received {} messages from Azure for partition {}.", batchSize, partitionId); } catch (Exception e) { logger.warn("Failed to receive messages: {}", e.getMessage()); } } }
@Override protected EventData fromByte(byte[] payload) { return EventData.create(payload); } }
@Override public Map<String, Object> getProperties() { return eventData.getProperties(); }
@Override public Object getObject() { return eventData.getObject(); }
private void updateMetrics(EventData event) { int eventDataLength = event.getBytes() == null ? 0 : event.getBytes().length; eventReadRate.inc(); aggEventReadRate.inc(); eventByteReadRate.inc(eventDataLength); aggEventByteReadRate.inc(eventDataLength); long latencyMs = Duration.between(event.getSystemProperties().getEnqueuedTime(), Instant.now()).toMillis(); readLatency.update(latencyMs); aggConsumptionLagMs.update(latencyMs); }
for (final EventData eventData : messages) { try (final InputStream in = new ByteArrayInputStream(eventData.getBytes())) { final RecordReader reader = readerFactory.createRecordReader(schemaRetrievalVariables, in, logger); session.write(failed, o -> o.write(eventData.getBytes())); putEventHubAttributes(attributes, eventHubName, partitionId, eventData); failed = session.putAllAttributes(failed, attributes);
@Override public EventDataWrap receive() { EventDataWrap eventDatawrap = receiver.receive(); if (eventDatawrap != null) { lastOffset = eventDatawrap.getEventData().getSystemProperties().getOffset(); } return eventDatawrap; }
public static String toString(EventData eventData) { Map<String, Object> map = new LinkedHashMap<>(); map.put("body", eventData.getObject()); map.put("offset", eventData.getSystemProperties().getOffset()); map.put("sequenceNumber", eventData.getSystemProperties().getSequenceNumber()); map.put("enqueuedTime", eventData.getSystemProperties().getEnqueuedTime()); return map.toString(); } }
@Override public void accept(PartitionReceiver receiver) { while (true) { try { Iterable<EventData> receivedEvents = receiver.receive(MAX_MESSAGES).get(); int batchSize = 0; if (receivedEvents != null) { for (EventData receivedEvent : receivedEvents) { String deviceId = (String) receivedEvent.getProperties().get("iothub-connection-device-id"); Map<String, Object> deviceState = null; try { deviceState = ParaObjectUtils.getJsonReader(Map.class).readValue(receivedEvent.getBytes()); logger.debug("Message received from Azure: {}", deviceState); } catch (Exception e) { } if (deviceState != null) { Thing t = thingFromCloudID(deviceId); if (t != null) { t.setDeviceState(deviceState); Para.getDAO().update(t.getAppid(), t); } } batchSize++; } } logger.debug("Received {} messages from Azure for partition {}.", batchSize, partitionId); } catch (Exception e) { logger.warn("Failed to receive messages: {}", e.getMessage()); } } }
@Override protected EventData fromByte(byte[] payload) { return EventData.create(payload); } }
protected EventData createEventData(String streamId, OutgoingMessageEnvelope envelope) { Optional<Interceptor> interceptor = Optional.ofNullable(interceptors.getOrDefault(streamId, null)); byte[] eventValue = (byte[]) envelope.getMessage(); if (interceptor.isPresent()) { eventValue = interceptor.get().intercept(eventValue); } EventData eventData = new EventDataImpl(eventValue); eventData.getProperties().put(PRODUCE_TIMESTAMP, Long.toString(System.currentTimeMillis())); if (config.getSendKeyInEventProperties(systemName)) { String keyValue = ""; if (envelope.getKey() != null) { keyValue = (envelope.getKey() instanceof byte[]) ? new String((byte[]) envelope.getKey()) : envelope.getKey().toString(); } eventData.getProperties().put(KEY, keyValue); } return eventData; }