/** * Updates the Diagnostics Metrics (processing time and number of events) * @param beginProcessing when sendOutput Started processing this message * @param endProcessing when sendOutput finished processing this message * @param metadata the event's message metadata */ private void updateMetrics(Instant beginProcessing, Instant endProcessing, SamzaSqlRelMsgMetadata metadata) { /* insert (SendToOutputStream) metrics */ insertProcessingTime.update(Duration.between(beginProcessing, endProcessing).toMillis()); /* query metrics */ Instant outputTime = Instant.now(); queryOutputEvents.inc(); /* TODO: remove scanTime validation once code to assign it is stable */ Validate.isTrue(metadata.hasScanTime()); Instant scanTime = Instant.parse(metadata.getscanTime()); queryLatency.update(Duration.between(scanTime, outputTime).toMillis()); /** TODO: change if hasArrivalTime to validation once arrivalTime is assigned, and later remove the check once code is stable */ if (metadata.hasArrivalTime()) { Instant arrivalTime = Instant.parse(metadata.getarrivalTime()); queueingLatency.update(Duration.between(arrivalTime, scanTime).toMillis()); } /* since availability of eventTime depends on source, we need the following check */ if (metadata.hasEventTime()) { Instant eventTime = Instant.parse(metadata.getEventTime()); totalLatency.update(Duration.between(eventTime, outputTime).toMillis()); } } } // OutputMapFunction
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex, MessageStream<SamzaSqlRelMessage> inputStream) { return inputStream.flatMap(message -> { Object field = message.getSamzaSqlRelRecord().getFieldValues().get(flattenIndex); if (field != null && field instanceof List) { List<SamzaSqlRelMessage> outMessages = new ArrayList<>(); SamzaSqlRelMsgMetadata messageMetadata = message.getSamzaSqlRelMsgMetadata(); SamzaSqlRelMsgMetadata newMetadata = new SamzaSqlRelMsgMetadata(messageMetadata.getEventTime(), messageMetadata.getarrivalTime(), messageMetadata.getscanTime(), true); for (Object fieldValue : (List) field) { List<Object> newValues = new ArrayList<>(message.getSamzaSqlRelRecord().getFieldValues()); newValues.set(flattenIndex, Collections.singletonList(fieldValue)); outMessages.add(new SamzaSqlRelMessage(message.getSamzaSqlRelRecord().getFieldNames(), newValues, newMetadata)); newMetadata = new SamzaSqlRelMsgMetadata(newMetadata.getEventTime(), newMetadata.getarrivalTime(), newMetadata.getscanTime(), false); } return outMessages; } else { message.getSamzaSqlRelMsgMetadata().isNewInputMessage = true; return Collections.singletonList(message); } }); }