/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. * @param metadata the message/event's metadata */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord, @JsonProperty("samzaSqlRelMsgMetadata") SamzaSqlRelMsgMetadata metadata) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues(), metadata); }
/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues()); }
@Override public Object getMessageKey(SamzaSqlRelMessage message) { SamzaSqlRelRecord keyRecord = getMessageKeyRelRecord(message); // If all the message key rel record values are null, return null message key. if (keyRecord.getFieldValues().stream().allMatch(Objects::isNull)) { return null; } // Using the table key converter, convert message key from rel format to the record key format. return relTableKeyConverter.convertToTableKeyFormat(keyRecord); }
/** * Create the SamzaSqlCompositeKey from the rel message. * @param message Represents the samza sql rel message. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlCompositeKey createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { ArrayList<Object> keyParts = new ArrayList<>(); for (int idx : relIdx) { keyParts.add(message.getSamzaSqlRelRecord().getFieldValues().get(idx)); } return new SamzaSqlCompositeKey(keyParts); } }
@Override protected List<Object> getTableRelRecordFieldValues(KV record) { // Using the message rel converter, convert message to sql rel message and add to output values. SamzaSqlRelMessage relMessage = msgConverter.convertToRelMessage(record); return relMessage.getSamzaSqlRelRecord().getFieldValues(); }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values from. * @param keyValueIdx list of key values in the form of field indices within the rel message. * @param keyPartNames Represents the key field names. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> keyValueIdx, List<String> keyPartNames) { Validate.isTrue(keyValueIdx.size() == keyPartNames.size(), "Key part name and value list sizes are different"); ArrayList<Object> keyPartValues = new ArrayList<>(); for (int idx : keyValueIdx) { keyPartValues.add(message.getSamzaSqlRelRecord().getFieldValues().get(idx)); } return new SamzaSqlRelRecord(keyPartNames, keyPartValues); }
@Override public boolean apply(SamzaSqlRelMessage message) { Object[] result = new Object[1]; expr.execute(context.getExecutionContext(), context.getDataContext(), message.getSamzaSqlRelRecord().getFieldValues().toArray(), result); if (result.length > 0 && result[0] instanceof Boolean) { boolean retVal = (Boolean) result[0]; log.debug( String.format("return value for input %s is %s", Arrays.asList(message.getSamzaSqlRelRecord().getFieldValues()).toString(), retVal)); return retVal; } else { log.error("return value is not boolean"); return false; } } }
@Override public Object convertToTableKeyFormat(SamzaSqlRelRecord relRecord) { if (relRecord.getFieldValues().get(0) instanceof SamzaSqlRelRecord) { relRecord = (SamzaSqlRelRecord) relRecord.getFieldValues().get(0); } return relRecord.getFieldValues().stream().map(Object::toString).collect(Collectors.toList()).get(0); } }
@Override protected List<Object> getTableRelRecordFieldValues(KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record) { return record.getValue().getSamzaSqlRelRecord().getFieldValues(); }
private GenericRecord convertToGenericRecord(SamzaSqlRelRecord relRecord, Schema schema) { GenericRecord record = new GenericData.Record(schema); List<String> fieldNames = relRecord.getFieldNames(); List<Object> values = relRecord.getFieldValues(); for (int index = 0; index < fieldNames.size(); index++) { if (!fieldNames.get(index).equalsIgnoreCase(SamzaSqlRelMessage.KEY_NAME)) { Object relObj = values.get(index); String fieldName = fieldNames.get(index); Schema fieldSchema = schema.getField(fieldName).schema(); record.put(fieldName, convertToAvroObject(relObj, getNonNullUnionSchema(fieldSchema))); } } return record; }
@Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message) { RelDataType type = project.getRowType(); Object[] output = new Object[type.getFieldCount()]; expr.execute(context.getExecutionContext(), context.getDataContext(), message.getSamzaSqlRelRecord().getFieldValues().toArray(), output); List<String> names = new ArrayList<>(); for (int index = 0; index < output.length; index++) { names.add(index, project.getNamedProjects().get(index).getValue()); } return new SamzaSqlRelMessage(names, Arrays.asList(output)); } }
@Test public void testNoArgs() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute(); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(new ArrayList<>(), new ArrayList<>()); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testNestedRecord() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord nestedSamzaSqlRelRecord = new SamzaSqlRelRecord(Arrays.asList("k3"), Arrays.asList("v3")); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", nestedSamzaSqlRelRecord); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", nestedSamzaSqlRelRecord)); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testMultiPairs() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", "v2"); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", "v2")); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testSinglePair() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("key", "value"); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("key"), Arrays.asList("value")); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testCompositeKeyCreation() { List<String> keyPartNames = Arrays.asList("kfield1", "kfield2"); SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Collections.singletonList(0)); Assert.assertEquals(relRecord1.getFieldNames().size(), 1); Assert.assertEquals(relRecord1.getFieldNames().get(0), "field1"); Assert.assertEquals(relRecord1.getFieldValues().get(0), "value1"); SamzaSqlRelRecord relRecord2 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1, 0))); Assert.assertEquals(relRecord2.getFieldNames().size(), 2); Assert.assertEquals(relRecord2.getFieldNames().get(0), "kfield2"); Assert.assertEquals(relRecord2.getFieldValues().get(0), "value2"); Assert.assertEquals(relRecord2.getFieldNames().get(1), "kfield1"); Assert.assertEquals(relRecord2.getFieldValues().get(1), "value1"); }
@Test public void testEmptyRecordConversion() { GenericData.Record record = new GenericData.Record(SimpleRecord.SCHEMA$); SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record)); Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), message.getSamzaSqlRelRecord().getFieldValues().size()); }
@Test public void testNullRecordConversion() { SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", null)); Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), message.getSamzaSqlRelRecord().getFieldValues().size()); }
@Test public void testWithDifferentFields() { SamzaSqlRelRecord record = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")).getSamzaSqlRelRecord(); SamzaSqlRelRecordSerde serde = (SamzaSqlRelRecordSerde) new SamzaSqlRelRecordSerdeFactory().getSerde(null, null); SamzaSqlRelRecord resultRecord = serde.fromBytes(serde.toBytes(record)); Assert.assertEquals(names, resultRecord.getFieldNames()); Assert.assertEquals(values, resultRecord.getFieldValues()); }
@Test public void testWithDifferentFields() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessageSerde serde = (SamzaSqlRelMessageSerde) new SamzaSqlRelMessageSerdeFactory().getSerde(null, null); SamzaSqlRelMessage resultMsg = serde.fromBytes(serde.toBytes(message)); Assert.assertEquals(names, resultMsg.getSamzaSqlRelRecord().getFieldNames()); Assert.assertEquals(values, resultMsg.getSamzaSqlRelRecord().getFieldValues()); }