/** * update metrics given a message * @param message the input message * @return the same message */ @Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message) { inputEvents.inc(); message.getSamzaSqlRelMsgMetadata().operatorBeginProcessingInstant = Instant.now().toString(); return message; }
/** * update metrics given a message * @param message the input message * @return the same message */ @Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message) { Instant endProcessing = Instant.now(); Instant beginProcessing = Instant.parse(message.getSamzaSqlRelMsgMetadata().operatorBeginProcessingInstant); outputEvents.inc(); processingTime.update(Duration.between(beginProcessing, endProcessing).toMillis()); return message; } } // TranslatorOutputMetricsMapFunction
@Override public KV<Object, Object> apply(SamzaSqlRelMessage message) { Instant beginProcessing = Instant.now(); KV<Object, Object> retKV = this.samzaMsgConverter.convertToSamzaMessage(message); updateMetrics(beginProcessing, Instant.now(), message.getSamzaSqlRelMsgMetadata()); return retKV; }
/** * transforms the input message into the output message with projected fields * @param message the input message to be transformed * @return the new SamzaSqlRelMessage message */ @Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message) { Instant arrivalTime = Instant.now(); RelDataType type = project.getRowType(); Object[] output = new Object[type.getFieldCount()]; expr.execute(translatorContext.getExecutionContext(), translatorContext.getDataContext(), message.getSamzaSqlRelRecord().getFieldValues().toArray(), output); List<String> names = new ArrayList<>(); for (int index = 0; index < output.length; index++) { names.add(index, project.getNamedProjects().get(index).getValue()); } updateMetrics(arrivalTime, Instant.now(), message.getSamzaSqlRelMsgMetadata().isNewInputMessage); return new SamzaSqlRelMessage(names, Arrays.asList(output), message.getSamzaSqlRelMsgMetadata()); }
@Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message, R record) { if (joinRelType.compareTo(JoinRelType.INNER) == 0 && record == null) { log.debug("Inner Join: Record not found for the message with key: " + getMessageKey(message)); // Returning null would result in Join operator implementation to filter out the message. return null; } // The resulting join output should be a SamzaSqlRelMessage containing the fields from both the stream message and // table record. The order of stream message fields and table record fields are dictated by the position of stream // and table in the 'from' clause of sql query. The output should also include the keys from both the stream message // and the table record. List<Object> outFieldValues = new ArrayList<>(); // If table position is on the right, add the stream message fields first if (isTablePosOnRight) { outFieldValues.addAll(message.getSamzaSqlRelRecord().getFieldValues()); } // Add the table record fields. if (record != null) { outFieldValues.addAll(getTableRelRecordFieldValues(record)); } else { // Table record could be null as the record could not be found in the store. This can // happen for outer joins. Add nulls to all the field values in the output message. tableFieldNames.forEach(s -> outFieldValues.add(null)); } // If table position is on the left, add the stream message fields last if (!isTablePosOnRight) { outFieldValues.addAll(message.getSamzaSqlRelRecord().getFieldValues()); } return new SamzaSqlRelMessage(outFieldNames, outFieldValues, message.getSamzaSqlRelMsgMetadata()); }
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex, MessageStream<SamzaSqlRelMessage> inputStream) { return inputStream.flatMap(message -> { Object field = message.getSamzaSqlRelRecord().getFieldValues().get(flattenIndex); if (field != null && field instanceof List) { List<SamzaSqlRelMessage> outMessages = new ArrayList<>(); SamzaSqlRelMsgMetadata messageMetadata = message.getSamzaSqlRelMsgMetadata(); SamzaSqlRelMsgMetadata newMetadata = new SamzaSqlRelMsgMetadata(messageMetadata.getEventTime(), messageMetadata.getarrivalTime(), messageMetadata.getscanTime(), true); for (Object fieldValue : (List) field) { List<Object> newValues = new ArrayList<>(message.getSamzaSqlRelRecord().getFieldValues()); newValues.set(flattenIndex, Collections.singletonList(fieldValue)); outMessages.add(new SamzaSqlRelMessage(message.getSamzaSqlRelRecord().getFieldNames(), newValues, newMetadata)); newMetadata = new SamzaSqlRelMsgMetadata(newMetadata.getEventTime(), newMetadata.getarrivalTime(), newMetadata.getscanTime(), false); } return outMessages; } else { message.getSamzaSqlRelMsgMetadata().isNewInputMessage = true; return Collections.singletonList(message); } }); }