/** * Constructs a new SamzaSqMessage given the input arguments * @param keyAndMessageKV * @param metadata * @return */ public static SamzaSqlInputMessage of (KV<Object, Object> keyAndMessageKV, SamzaSqlRelMsgMetadata metadata) { return new SamzaSqlInputMessage(keyAndMessageKV, metadata); }
@Override public boolean apply(SamzaSqlInputMessage samzaSqlInputMessage) { return !relConverter.isSystemMessage(samzaSqlInputMessage.getKeyAndMessageKV()); } }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values and names from. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { return createSamzaSqlCompositeKey(message, relIdx, getSamzaSqlCompositeKeyFieldNames(message.getSamzaSqlRelRecord().getFieldNames(), relIdx)); }
@Test public void testNestedRecord() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord nestedSamzaSqlRelRecord = new SamzaSqlRelRecord(Arrays.asList("k3"), Arrays.asList("v3")); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", nestedSamzaSqlRelRecord); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", nestedSamzaSqlRelRecord)); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Override protected List<Object> getTableRelRecordFieldValues(KV record) { // Using the message rel converter, convert message to sql rel message and add to output values. SamzaSqlRelMessage relMessage = msgConverter.convertToRelMessage(record); return relMessage.getSamzaSqlRelRecord().getFieldValues(); }
/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues()); }
/** * Create the SamzaSqlCompositeKey from the rel message. * @param message Represents the samza sql rel message. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlCompositeKey createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { ArrayList<Object> keyParts = new ArrayList<>(); for (int idx : relIdx) { keyParts.add(message.getSamzaSqlRelRecord().getFieldValues().get(idx)); } return new SamzaSqlCompositeKey(keyParts); } }
@Override protected List<Object> getTableRelRecordFieldValues(KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record) { return record.getValue().getSamzaSqlRelRecord().getFieldValues(); }
@Test public void testInEquality() { SamzaSqlRelRecord relRecord1 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, "object")); SamzaSqlRelRecord relRecord2 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, null)); assertNotEquals(relRecord1, relRecord2); assertNotEquals(relRecord1.hashCode(), relRecord2.hashCode()); } }
@SamzaSqlUdfMethod public SamzaSqlRelRecord execute(Object... args) { int numOfArgs = args.length; Validate.isTrue(numOfArgs % 2 == 0, "numOfArgs should be an even number"); List<String> fieldNames = new ArrayList<>(); List<Object> fieldValues = new ArrayList<>(); for (int i = 0; i < numOfArgs - 1; i += 2) { fieldNames.add((String) args[i]); // value can be instanceof SamzaSqlRelRecord, or any Object(string, int, long most likely) fieldValues.add(args[i + 1]); } return new SamzaSqlRelRecord(fieldNames, fieldValues); } }
@Override public Object getMessageKey(SamzaSqlRelMessage message) { SamzaSqlRelRecord keyRecord = getMessageKeyRelRecord(message); // If all the message key rel record values are null, return null message key. if (keyRecord.getFieldValues().stream().allMatch(Objects::isNull)) { return null; } // Using the table key converter, convert message key from rel format to the record key format. return relTableKeyConverter.convertToTableKeyFormat(keyRecord); }
/** * Creates a {@link SamzaSqlRelMessage} from the list of relational fields and values. * If the field list contains KEY, then it extracts the key out of the fields to create a * {@link SamzaSqlRelRecord} along with key, otherwise creates a {@link SamzaSqlRelRecord} * without the key. * @param fieldNames Ordered list of field names in the row. * @param fieldValues Ordered list of all the values in the row. Some of the fields can be null, This could be * result of delete change capture event in the stream or because of the result of the outer join * or the fields themselves are null in the original stream. */ public SamzaSqlRelMessage(List<String> fieldNames, List<Object> fieldValues) { Validate.isTrue(fieldNames.size() == fieldValues.size(), "Field Names and values are not of same length."); int keyIndex = fieldNames.indexOf(KEY_NAME); Object key = null; if (keyIndex != -1) { key = fieldValues.get(keyIndex); } this.key = key; this.samzaSqlRelRecord = new SamzaSqlRelRecord(fieldNames, fieldValues); }
@Override public Object convertToTableKeyFormat(SamzaSqlRelRecord relRecord) { if (relRecord.getFieldValues().get(0) instanceof SamzaSqlRelRecord) { relRecord = (SamzaSqlRelRecord) relRecord.getFieldValues().get(0); } return relRecord.getFieldValues().stream().map(Object::toString).collect(Collectors.toList()).get(0); } }
@Override public Object apply(IncomingMessageEnvelope ime) { Assert.notNull(ime, "ime is null"); KV<Object, Object> keyAndMessageKV = KV.of(ime.getKey(), ime.getMessage()); SamzaSqlRelMsgMetadata metadata = new SamzaSqlRelMsgMetadata(Instant.ofEpochMilli(ime.getEventTime()).toString(), Instant.ofEpochMilli(ime.getArrivalTime()).toString(), null); SamzaSqlInputMessage samzaMsg = SamzaSqlInputMessage.of(keyAndMessageKV, metadata); return samzaMsg; } }
@Test public void testMultiPairs() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", "v2"); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", "v2")); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. * @param metadata the message/event's metadata */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord, @JsonProperty("samzaSqlRelMsgMetadata") SamzaSqlRelMsgMetadata metadata) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues(), metadata); }
@Test public void testEquality() { SamzaSqlRelRecord relRecord1 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, "object")); SamzaSqlRelRecord relRecord2 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, "object")); assertEquals(relRecord1, relRecord2); assertEquals(relRecord1.hashCode(), relRecord2.hashCode()); }
/** * Creates a {@link SamzaSqlRelMessage} from the list of relational fields and values. * If the field list contains KEY, then it extracts the key out of the fields to create a * {@link SamzaSqlRelRecord} along with key, otherwise creates a {@link SamzaSqlRelRecord} * without the key. * @param fieldNames Ordered list of field names in the row. * @param fieldValues Ordered list of all the values in the row. Some of the fields can be null, This could be * result of delete change capture event in the stream or because of the result of the outer join * or the fields themselves are null in the original stream. * @param metadata the message/event's metadata */ public SamzaSqlRelMessage(List<String> fieldNames, List<Object> fieldValues, SamzaSqlRelMsgMetadata metadata) { Validate.isTrue(fieldNames.size() == fieldValues.size(), "Field Names and values are not of same length."); Validate.notNull(metadata, "Message metadata is NULL"); int keyIndex = fieldNames.indexOf(KEY_NAME); Object key = null; if (keyIndex != -1) { key = fieldValues.get(keyIndex); } this.key = key; this.samzaSqlRelRecord = new SamzaSqlRelRecord(fieldNames, fieldValues); this.samzaSqlRelMsgMetadata = metadata; }
@Test public void testSinglePair() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("key", "value"); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("key"), Arrays.asList("value")); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testNoArgs() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute(); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(new ArrayList<>(), new ArrayList<>()); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }