/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. * @param metadata the message/event's metadata */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord, @JsonProperty("samzaSqlRelMsgMetadata") SamzaSqlRelMsgMetadata metadata) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues(), metadata); }
/** * Creates the SamzaSqlRelMessage from {@link SamzaSqlRelRecord}. * @param samzaSqlRelRecord represents the rel record. */ public SamzaSqlRelMessage(@JsonProperty("samzaSqlRelRecord") SamzaSqlRelRecord samzaSqlRelRecord) { this(samzaSqlRelRecord.getFieldNames(), samzaSqlRelRecord.getFieldValues()); }
static Object extractField(String fieldName, Object current) { if (current instanceof SamzaSqlRelRecord) { SamzaSqlRelRecord record = (SamzaSqlRelRecord) current; Validate.isTrue(record.getFieldNames().contains(fieldName), String.format("Invalid field %s in %s", fieldName, record)); return record.getField(fieldName).orElse(null); } else if (current instanceof Map) { Map map = (Map) current; Validate.isTrue(map.containsKey(fieldName), String.format("Invalid field %s in %s", fieldName, map)); return map.get(fieldName); } else if (current instanceof List && fieldName.endsWith("]")) { List list = (List) current; int index = Integer.parseInt(fieldName.substring(fieldName.indexOf("[") + 1, fieldName.length() - 1)); return list.get(index); } throw new IllegalArgumentException(String.format( "Unsupported accessing operation for data type: %s with field: %s.", current.getClass(), fieldName)); } }
private GenericRecord convertToGenericRecord(SamzaSqlRelRecord relRecord, Schema schema) { GenericRecord record = new GenericData.Record(schema); List<String> fieldNames = relRecord.getFieldNames(); List<Object> values = relRecord.getFieldValues(); for (int index = 0; index < fieldNames.size(); index++) { if (!fieldNames.get(index).equalsIgnoreCase(SamzaSqlRelMessage.KEY_NAME)) { Object relObj = values.get(index); String fieldName = fieldNames.get(index); Schema fieldSchema = schema.getField(fieldName).schema(); record.put(fieldName, convertToAvroObject(relObj, getNonNullUnionSchema(fieldSchema))); } } return record; }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values and names from. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { return createSamzaSqlCompositeKey(message, relIdx, getSamzaSqlCompositeKeyFieldNames(message.getSamzaSqlRelRecord().getFieldNames(), relIdx)); }
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex, MessageStream<SamzaSqlRelMessage> inputStream) { return inputStream.flatMap(message -> { Object field = message.getSamzaSqlRelRecord().getFieldValues().get(flattenIndex); if (field != null && field instanceof List) { List<SamzaSqlRelMessage> outMessages = new ArrayList<>(); for (Object fieldValue : (List) field) { List<Object> newValues = new ArrayList<>(message.getSamzaSqlRelRecord().getFieldValues()); newValues.set(flattenIndex, Collections.singletonList(fieldValue)); outMessages.add(new SamzaSqlRelMessage(message.getSamzaSqlRelRecord().getFieldNames(), newValues)); } return outMessages; } else { return Collections.singletonList(message); } }); }
@Test public void testNoArgs() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute(); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(new ArrayList<>(), new ArrayList<>()); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testCompositeKeyCreation() { List<String> keyPartNames = Arrays.asList("kfield1", "kfield2"); SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Collections.singletonList(0)); Assert.assertEquals(relRecord1.getFieldNames().size(), 1); Assert.assertEquals(relRecord1.getFieldNames().get(0), "field1"); Assert.assertEquals(relRecord1.getFieldValues().get(0), "value1"); SamzaSqlRelRecord relRecord2 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1, 0))); Assert.assertEquals(relRecord2.getFieldNames().size(), 2); Assert.assertEquals(relRecord2.getFieldNames().get(0), "kfield2"); Assert.assertEquals(relRecord2.getFieldValues().get(0), "value2"); Assert.assertEquals(relRecord2.getFieldNames().get(1), "kfield1"); Assert.assertEquals(relRecord2.getFieldValues().get(1), "value1"); }
@Test public void testNestedRecord() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord nestedSamzaSqlRelRecord = new SamzaSqlRelRecord(Arrays.asList("k3"), Arrays.asList("v3")); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", nestedSamzaSqlRelRecord); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", nestedSamzaSqlRelRecord)); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex, MessageStream<SamzaSqlRelMessage> inputStream) { return inputStream.flatMap(message -> { Object field = message.getSamzaSqlRelRecord().getFieldValues().get(flattenIndex); if (field != null && field instanceof List) { List<SamzaSqlRelMessage> outMessages = new ArrayList<>(); SamzaSqlRelMsgMetadata messageMetadata = message.getSamzaSqlRelMsgMetadata(); SamzaSqlRelMsgMetadata newMetadata = new SamzaSqlRelMsgMetadata(messageMetadata.getEventTime(), messageMetadata.getarrivalTime(), messageMetadata.getscanTime(), true); for (Object fieldValue : (List) field) { List<Object> newValues = new ArrayList<>(message.getSamzaSqlRelRecord().getFieldValues()); newValues.set(flattenIndex, Collections.singletonList(fieldValue)); outMessages.add(new SamzaSqlRelMessage(message.getSamzaSqlRelRecord().getFieldNames(), newValues, newMetadata)); newMetadata = new SamzaSqlRelMsgMetadata(newMetadata.getEventTime(), newMetadata.getarrivalTime(), newMetadata.getscanTime(), false); } return outMessages; } else { message.getSamzaSqlRelMsgMetadata().isNewInputMessage = true; return Collections.singletonList(message); } }); }
@Test public void testMultiPairs() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("k1", "v1", "k2", "v2"); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("k1", "k2"), Arrays.asList("v1", "v2")); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testSinglePair() { BuildOutputRecordUdf buildOutputRecordUdf = new BuildOutputRecordUdf(); SamzaSqlRelRecord actualRecord = buildOutputRecordUdf.execute("key", "value"); SamzaSqlRelRecord expectedRecord = new SamzaSqlRelRecord(Arrays.asList("key"), Arrays.asList("value")); Assert.assertEquals(actualRecord.getFieldNames(), expectedRecord.getFieldNames()); Assert.assertEquals(actualRecord.getFieldValues(), expectedRecord.getFieldValues()); }
@Test public void testEmptyRecordConversion() { GenericData.Record record = new GenericData.Record(SimpleRecord.SCHEMA$); SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record)); Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), message.getSamzaSqlRelRecord().getFieldValues().size()); }
@Test public void testNullRecordConversion() { SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", null)); Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), message.getSamzaSqlRelRecord().getFieldValues().size()); }
outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(streamFieldNames); expectedFieldNames.addAll(tableFieldNames);
void translate(final LogicalAggregate aggregate, final TranslatorContext context) { validateAggregateFunctions(aggregate); MessageStream<SamzaSqlRelMessage> inputStream = context.getMessageStream(aggregate.getInput().getId()); // At this point, the assumption is that only count function is supported. SupplierFunction<Long> initialValue = () -> (long) 0; FoldLeftFunction<SamzaSqlRelMessage, Long> foldCountFn = (m, c) -> c + 1; final ArrayList<String> aggFieldNames = getAggFieldNames(aggregate); MessageStream<SamzaSqlRelMessage> outputStream = inputStream .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(context.getExecutionContext().getSamzaSqlApplicationConfig().getWindowDurationMs()), initialValue, foldCountFn, new SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde(), new LongSerde()) .setAccumulationMode(AccumulationMode.DISCARDING), "tumblingWindow_" + windowId) .map(windowPane -> { List<String> fieldNames = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldNames(); List<Object> fieldValues = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldValues(); fieldNames.add(aggFieldNames.get(0)); fieldValues.add(windowPane.getMessage()); return new SamzaSqlRelMessage(fieldNames, fieldValues); }); context.registerMessageStream(aggregate.getId(), outputStream); }
@Test public void testWithInnerJoinWithTableOnRight() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = new SamzaSqlRelMessage(tableFieldNames, tableFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 1); List<Integer> tableKeyIds = Arrays.asList(0, 1); SamzaSqlRelRecord compositeKey = SamzaSqlRelMessage.createSamzaSqlCompositeKey(tableMsg, tableKeyIds); KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record = KV.of(compositeKey, tableMsg); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(true); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(false); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(streamFieldNames); expectedFieldNames.addAll(tableFieldNames); List<Object> expectedFieldValues = new ArrayList<>(streamFieldValues); expectedFieldValues.addAll(tableFieldValues); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues); }
@Test public void testWithInnerJoinWithTableOnLeft() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = new SamzaSqlRelMessage(tableFieldNames, tableFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 2); List<Integer> tableKeyIds = Arrays.asList(0, 2); SamzaSqlRelRecord compositeKey = SamzaSqlRelMessage.createSamzaSqlCompositeKey(tableMsg, tableKeyIds); KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record = KV.of(compositeKey, tableMsg); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(false); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(true); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(tableFieldNames); expectedFieldNames.addAll(streamFieldNames); List<Object> expectedFieldValues = new ArrayList<>(tableFieldValues); expectedFieldValues.addAll(streamFieldValues); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues); }
@Test public void testWithDifferentFields() { SamzaSqlRelRecord record = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")).getSamzaSqlRelRecord(); SamzaSqlRelRecordSerde serde = (SamzaSqlRelRecordSerde) new SamzaSqlRelRecordSerdeFactory().getSerde(null, null); SamzaSqlRelRecord resultRecord = serde.fromBytes(serde.toBytes(record)); Assert.assertEquals(names, resultRecord.getFieldNames()); Assert.assertEquals(values, resultRecord.getFieldValues()); }
@Test public void testWithDifferentFields() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessageSerde serde = (SamzaSqlRelMessageSerde) new SamzaSqlRelMessageSerdeFactory().getSerde(null, null); SamzaSqlRelMessage resultMsg = serde.fromBytes(serde.toBytes(message)); Assert.assertEquals(names, resultMsg.getSamzaSqlRelRecord().getFieldNames()); Assert.assertEquals(values, resultMsg.getSamzaSqlRelRecord().getFieldValues()); }