@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); String messageData = ""; if (eventData.getBytes() != null) { messageData = new String(eventData.getBytes()); } /*Will only serialize AMQPValue type*/ else if (eventData.getObject() != null) { try { if (!(eventData.getObject() instanceof List)) { messageData = eventData.getObject().toString(); } else { throw new RuntimeException("Cannot serialize the given AMQP type."); } } catch (RuntimeException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw e; } } fieldContents.add(messageData); return fieldContents; }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); String messageData = ""; if (eventData.getBytes() != null) { messageData = new String(eventData.getBytes()); } /*Will only serialize AMQPValue type*/ else if (eventData.getObject() != null) { try { if (!(eventData.getObject() instanceof List)) { messageData = eventData.getObject().toString(); } else { throw new RuntimeException("Cannot serialize the given AMQP type"); } } catch (RuntimeException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw e; } } Map<String, Object> metaDataMap = eventData.getProperties(); fieldContents.add(messageData); fieldContents.add(metaDataMap); return fieldContents; }
@Override public List<Object> deserialize(EventData eventData) { final List<Object> fieldContents = new ArrayList<Object>(); byte[] messageData = null; if (eventData.getBytes() != null) { messageData = eventData.getBytes(); } else if (eventData.getObject() != null) { try { messageData = SerializeDeserializeUtil.serialize(eventData.getObject()); } catch (IOException e) { logger.error("Failed to serialize EventData payload class" + eventData.getObject().getClass()); logger.error("Exception encountered while serializing EventData payload is" + e.toString()); throw new RuntimeException(e); } } Map<String, Object> metaDataMap = eventData.getProperties(); Map<String, Object> systemMetaDataMap = eventData.getSystemProperties(); fieldContents.add(messageData); fieldContents.add(metaDataMap); fieldContents.add(systemMetaDataMap); return fieldContents; }
private void writeFlowFiles(PartitionContext context, Iterable<EventData> messages, ProcessSession session, StopWatch stopWatch) { final String eventHubName = context.getEventHubPath(); final String partitionId = context.getPartitionId(); final String consumerGroup = context.getConsumerGroupName(); messages.forEach(eventData -> { FlowFile flowFile = session.create(); final Map<String, String> attributes = new HashMap<>(); putEventHubAttributes(attributes, eventHubName, partitionId, eventData); flowFile = session.putAllAttributes(flowFile, attributes); flowFile = session.write(flowFile, out -> { out.write(eventData.getBytes()); }); transferTo(REL_SUCCESS, session, stopWatch, eventHubName, partitionId, consumerGroup, flowFile); }); }
out.write(eventData.getBytes()); });
for (final EventData eventData : messages) { try (final InputStream in = new ByteArrayInputStream(eventData.getBytes())) { final RecordReader reader = readerFactory.createRecordReader(schemaRetrievalVariables, in, logger); session.write(failed, o -> o.write(eventData.getBytes())); putEventHubAttributes(attributes, eventHubName, partitionId, eventData); failed = session.putAllAttributes(failed, attributes);
@Override protected byte[] getPayload(EventData azureMessage) { return azureMessage.getBytes(); }
@Override protected byte[] getPayload(EventData azureMessage) { return azureMessage.getBytes(); }
@Override public byte[] getBytes() { return eventData.getBytes(); }
private void updateMetrics(EventData event) { int eventDataLength = event.getBytes() == null ? 0 : event.getBytes().length; eventReadRate.inc(); aggEventReadRate.inc(); eventByteReadRate.inc(eventDataLength); aggEventByteReadRate.inc(eventDataLength); long latencyMs = Duration.between(event.getSystemProperties().getEnqueuedTime(), Instant.now()).toMillis(); readLatency.update(latencyMs); aggConsumptionLagMs.update(latencyMs); }
@Override public void onEvents(PartitionContext context, Iterable<EventData> messages) throws ExecutionException, InterruptedException { // don't issue a write until we checkpoint or exit this callback List<Span> buffer = new ArrayList<>(); for (EventData data : messages) { byte[] bytes = data.getBytes(); BytesDecoder<Span> decoder = decoderForListMessage(bytes); List<Span> nextSpans = decoder.decodeList(bytes); buffer.addAll(nextSpans); if (maybeCheckpoint(context, data, nextSpans.size())) { collector.accept(buffer, NOOP); buffer.clear(); } } if (!buffer.isEmpty()) { collector.accept(buffer, NOOP); } }
private void receiveMessages(PartitionReceiver receiver, int numMessages) throws EventHubException { int count = 0; while (count < numMessages) { Iterable<EventData> messages = receiver.receiveSync(100); if (messages == null) { break; } for (EventData data : messages) { count++; LOG.info("Data" + new String(data.getBytes())); } } }
@Override public void accept(PartitionReceiver receiver) { while (true) { try { Iterable<EventData> receivedEvents = receiver.receive(MAX_MESSAGES).get(); int batchSize = 0; if (receivedEvents != null) { for (EventData receivedEvent : receivedEvents) { String deviceId = (String) receivedEvent.getProperties().get("iothub-connection-device-id"); Map<String, Object> deviceState = null; try { deviceState = ParaObjectUtils.getJsonReader(Map.class).readValue(receivedEvent.getBytes()); logger.debug("Message received from Azure: {}", deviceState); } catch (Exception e) { } if (deviceState != null) { Thing t = thingFromCloudID(deviceId); if (t != null) { t.setDeviceState(deviceState); Para.getDAO().update(t.getAppid(), t); } } batchSize++; } } logger.debug("Received {} messages from Azure for partition {}.", batchSize, partitionId); } catch (Exception e) { logger.warn("Failed to receive messages: {}", e.getMessage()); } } }
@Override public void accept(PartitionReceiver receiver) { while (true) { try { Iterable<EventData> receivedEvents = receiver.receive(MAX_MESSAGES).get(); int batchSize = 0; if (receivedEvents != null) { for (EventData receivedEvent : receivedEvents) { String deviceId = (String) receivedEvent.getProperties().get("iothub-connection-device-id"); Map<String, Object> deviceState = null; try { deviceState = ParaObjectUtils.getJsonReader(Map.class).readValue(receivedEvent.getBytes()); logger.debug("Message received from Azure: {}", deviceState); } catch (Exception e) { } if (deviceState != null) { Thing t = thingFromCloudID(deviceId); if (t != null) { t.setDeviceState(deviceState); Para.getDAO().update(t.getAppid(), t); } } batchSize++; } } logger.debug("Received {} messages from Azure for partition {}.", batchSize, partitionId); } catch (Exception e) { logger.warn("Failed to receive messages: {}", e.getMessage()); } } }
@Override public void onReceive(Iterable<EventData> events) { if (events != null) { events.forEach(event -> { byte[] eventDataBody = event.getBytes(); if (interceptor != null) { eventDataBody = interceptor.intercept(eventDataBody); } String offset = event.getSystemProperties().getOffset(); Object partitionKey = event.getSystemProperties().getPartitionKey(); if (partitionKey == null) { partitionKey = event.getProperties().get(EventHubSystemProducer.KEY); } try { updateMetrics(event); // note that the partition key can be null put(ssp, new EventHubIncomingMessageEnvelope(ssp, offset, partitionKey, eventDataBody, event)); } catch (InterruptedException e) { String msg = String.format("Interrupted while adding the event from ssp %s to dispatch queue.", ssp); LOG.error(msg, e); throw new SamzaException(msg, e); } // Cache latest checkpoint streamPartitionOffsets.put(ssp, offset); }); } }
@Override public void onEvents(PartitionContext context, Iterable<EventData> events) throws Exception { LOGGER.info("Partition " + context.getPartitionId() + " got event batch"); int eventCount = 0; for (EventData data : events) { try { LOGGER.info("SAMPLE (" + context.getPartitionId() + "," + data.getSystemProperties().getOffset() + "," + data.getSystemProperties().getSequenceNumber() + "): " + new String(data.getBytes(), "UTF8")); eventCount++; this.checkpointBatchingCount++; if ((checkpointBatchingCount % 5) == 0) { LOGGER.info("SAMPLE: Partition " + context.getPartitionId() + " checkpointing at " + data.getSystemProperties().getOffset() + "," + data.getSystemProperties().getSequenceNumber()); context.checkpoint(data).get(); } } catch (Exception e) { LOGGER.error("Processing failed for an event: " + e.toString(), e); } } LOGGER.info("Partition " + context.getPartitionId() + " batch size was " + eventCount + " for host " + context.getOwner()); } }
.map(eventData -> new String(eventData.getBytes())) .collect(Collectors.toList());
.map(eventData -> new String(eventData.getBytes())) .collect(Collectors.toList());
private void verifyEvents(List<IncomingMessageEnvelope> messages, List<EventData> eventDataList, Interceptor interceptor) { Assert.assertEquals(messages.size(), eventDataList.size()); for (int i = 0; i < messages.size(); i++) { IncomingMessageEnvelope message = messages.get(i); EventData eventData = eventDataList.get(i); Assert.assertEquals(message.getKey(), eventData.getSystemProperties().getPartitionKey()); Assert.assertEquals(message.getMessage(), interceptor.intercept(eventData.getBytes())); Assert.assertEquals(message.getOffset(), eventData.getSystemProperties().getOffset()); } }