@Test public void testNestedRecord() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord nestedSamzaSqlRelRecord = new SamzaSqlRelRecord(Arrays.asList("k3"), Arrays.asList("v3")); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", nestedSamzaSqlRelRecord); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", nestedSamzaSqlRelRecord)); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
static Object extractField(String fieldName, Object current) { if (current instanceof SamzaSqlRelRecord) { SamzaSqlRelRecord record = (SamzaSqlRelRecord) current; Validate.isTrue(record.getFieldNames().contains(fieldName), String.format("Invalid field %s in %s", fieldName, record)); return record.getField(fieldName).orElse(null); } else if (current instanceof Map) { Map map = (Map) current; Validate.isTrue(map.containsKey(fieldName), String.format("Invalid field %s in %s", fieldName, map)); return map.get(fieldName); } else if (current instanceof List && fieldName.endsWith("]")) { List list = (List) current; int index = Integer.parseInt(fieldName.substring(fieldName.indexOf("[") + 1, fieldName.length() - 1)); return list.get(index); } throw new IllegalArgumentException(String.format( "Unsupported accessing operation for data type: %s with field: %s.", current.getClass(), fieldName)); } }
@Test public void testInEquality() { SamzaSqlRelRecord relRecord1 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, "object")); SamzaSqlRelRecord relRecord2 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, null)); assertNotEquals(relRecord1, relRecord2); assertNotEquals(relRecord1.hashCode(), relRecord2.hashCode()); } }
/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues()); }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values from. * @param keyValueIdx list of key values in the form of field indices within the rel message. * @param keyPartNames Represents the key field names. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> keyValueIdx, List<String> keyPartNames) { Validate.isTrue(keyValueIdx.size() == keyPartNames.size(), "Key part name and value list sizes are different"); ArrayList<Object> keyPartValues = new ArrayList<>(); for (int idx : keyValueIdx) { keyPartValues.add(message.getSamzaSqlRelRecord().getFieldValues().get(idx)); } return new SamzaSqlRelRecord(keyPartNames, keyPartValues); }
/** * Creates a {@link SamzaSqlRelMessage} from the list of relational fields and values. * If the field list contains KEY, then it extracts the key out of the fields to create a * {@link SamzaSqlRelRecord} along with key, otherwise creates a {@link SamzaSqlRelRecord} * without the key. * @param fieldNames Ordered list of field names in the row. * @param fieldValues Ordered list of all the values in the row. Some of the fields can be null, This could be * result of delete change capture event in the stream or because of the result of the outer join * or the fields themselves are null in the original stream. */ public SamzaSqlRelMessage(List<String> fieldNames, List<Object> fieldValues) { Validate.isTrue(fieldNames.size() == fieldValues.size(), "Field Names and values are not of same length."); int keyIndex = fieldNames.indexOf(KEY_NAME); Object key = null; if (keyIndex != -1) { key = fieldValues.get(keyIndex); } this.key = key; this.samzaSqlRelRecord = new SamzaSqlRelRecord(fieldNames, fieldValues); }
@Override public Object getMessageKey(SamzaSqlRelMessage message) { SamzaSqlRelRecord keyRecord = getMessageKeyRelRecord(message); // If all the message key rel record values are null, return null message key. if (keyRecord.getFieldValues().stream().allMatch(Objects::isNull)) { return null; } // Using the table key converter, convert message key from rel format to the record key format. return relTableKeyConverter.convertToTableKeyFormat(keyRecord); }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values and names from. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { return createSamzaSqlCompositeKey(message, relIdx, getSamzaSqlCompositeKeyFieldNames(message.getSamzaSqlRelRecord().getFieldNames(), relIdx)); }
@Test public void testGetField() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertEquals(values.get(0), message.getSamzaSqlRelRecord().getField(names.get(0)).get()); Assert.assertEquals(values.get(1), message.getSamzaSqlRelRecord().getField(names.get(1)).get()); }
/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. * @param metadata the message/event's metadata */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord, @JsonProperty("samzaSqlRelMsgMetadata") SamzaSqlRelMsgMetadata metadata) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues(), metadata); }
@SamzaSqlUdfMethod public SamzaSqlRelRecord execute(Object... args) { int numOfArgs = args.length; Validate.isTrue(numOfArgs % 2 == 0, "numOfArgs should be an even number"); List<String> fieldNames = new ArrayList<>(); List<Object> fieldValues = new ArrayList<>(); for (int i = 0; i < numOfArgs - 1; i += 2) { fieldNames.add((String) args[i]); // value can be instanceof SamzaSqlRelRecord, or any Object(string, int, long most likely) fieldValues.add(args[i + 1]); } return new SamzaSqlRelRecord(fieldNames, fieldValues); } }
/** * Create the SamzaSqlCompositeKey from the rel message. * @param message Represents the samza sql rel message. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlCompositeKey createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { ArrayList<Object> keyParts = new ArrayList<>(); for (int idx : relIdx) { keyParts.add(message.getSamzaSqlRelRecord().getFieldValues().get(idx)); } return new SamzaSqlCompositeKey(keyParts); } }
@Test public void testGetNonExistentField() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertFalse(message.getSamzaSqlRelRecord().getField("field3").isPresent()); }
private GenericRecord convertToGenericRecord(SamzaSqlRelRecord relRecord, Schema schema) { GenericRecord record = new GenericData.Record(schema); List<String> fieldNames = relRecord.getFieldNames(); List<Object> values = relRecord.getFieldValues(); for (int index = 0; index < fieldNames.size(); index++) { if (!fieldNames.get(index).equalsIgnoreCase(SamzaSqlRelMessage.KEY_NAME)) { Object relObj = values.get(index); String fieldName = fieldNames.get(index); Schema fieldSchema = schema.getField(fieldName).schema(); record.put(fieldName, convertToAvroObject(relObj, getNonNullUnionSchema(fieldSchema))); } } return record; }
@Test public void testNoArgs() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute(); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(new ArrayList<>(), new ArrayList<>()); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
/** * Create the SamzaSqlRelMessage, Each rel message represents a row in the table. * So it can contain a key and a list of fields in the row. * @param key Represents the key in the row, Key can be null. * @param fieldNames Ordered list of field names in the row. * @param fieldValues Ordered list of all the values in the row. Some of the fields can be null, This could be result of * delete change capture event in the stream or because of the result of the outer join or the fields * themselves are null in the original stream. */ public SamzaSqlRelMessage(Object key, List<String> fieldNames, List<Object> fieldValues) { Validate.isTrue(fieldNames.size() == fieldValues.size(), "Field Names and values are not of same length."); List<String> tmpFieldNames = new ArrayList<>(); List<Object> tmpFieldValues = new ArrayList<>(); this.key = key; tmpFieldNames.add(KEY_NAME); tmpFieldValues.add(key); tmpFieldNames.addAll(fieldNames); tmpFieldValues.addAll(fieldValues); this.samzaSqlRelRecord = new SamzaSqlRelRecord(tmpFieldNames, tmpFieldValues); }
@Override protected List<Object> getTableRelRecordFieldValues(KV record) { // Using the message rel converter, convert message to sql rel message and add to output values. SamzaSqlRelMessage relMessage = msgConverter.convertToRelMessage(record); return relMessage.getSamzaSqlRelRecord().getFieldValues(); }
@Test public void testEquality() { SamzaSqlRelRecord relRecord1 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, "object")); SamzaSqlRelRecord relRecord2 = new SamzaSqlRelRecord(Arrays.asList("id", "name"), Arrays.asList(1L, "object")); assertEquals(relRecord1, relRecord2); assertEquals(relRecord1.hashCode(), relRecord2.hashCode()); }
Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), ComplexRecord.SCHEMA$.getFields().size() + 1); Assert.assertEquals(message.getSamzaSqlRelRecord().getField("id").get(), id); Assert.assertEquals(message.getSamzaSqlRelRecord().getField("bool_value").get(), boolValue); Assert.assertEquals(message.getSamzaSqlRelRecord().getField("double_value").get(), doubleValue); Assert.assertEquals(message.getSamzaSqlRelRecord().getField("string_value").get(), new Utf8(testStrValue)); Assert.assertEquals(message.getSamzaSqlRelRecord().getField("float_value").get(), doubleValue); Assert.assertEquals(message.getSamzaSqlRelRecord().getField("long_value").get(), longValue); Assert.assertTrue( arrayValue.stream() .map(Utf8::new) .collect(Collectors.toList()) .equals(message.getSamzaSqlRelRecord().getField("array_values").get())); Assert.assertTrue(mapValue.entrySet() .stream() .collect(Collectors.toMap(x -> new Utf8(x.getKey()), y -> new Utf8(y.getValue()))) .equals(message.getSamzaSqlRelRecord().getField("map_values").get())); Arrays.equals(((ByteString) message.getSamzaSqlRelRecord().getField("bytes_value").get()).getBytes(), testBytes.array())); Assert.assertTrue( Arrays.equals(((ByteString) message.getSamzaSqlRelRecord().getField("fixed_value").get()).getBytes(), DEFAULT_TRACKING_ID_BYTES));
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex, MessageStream<SamzaSqlRelMessage> inputStream) { return inputStream.flatMap(message -> { Object field = message.getSamzaSqlRelRecord().getFieldValues().get(flattenIndex); if (field != null && field instanceof List) { List<SamzaSqlRelMessage> outMessages = new ArrayList<>(); for (Object fieldValue : (List) field) { List<Object> newValues = new ArrayList<>(message.getSamzaSqlRelRecord().getFieldValues()); newValues.set(flattenIndex, Collections.singletonList(fieldValue)); outMessages.add(new SamzaSqlRelMessage(message.getSamzaSqlRelRecord().getFieldNames(), newValues)); } return outMessages; } else { return Collections.singletonList(message); } }); }