/** * Create the SamzaSqlCompositeKey from the rel message. * @param message Represents the samza sql rel message. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlCompositeKey createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { ArrayList<Object> keyParts = new ArrayList<>(); for (int idx : relIdx) { keyParts.add(message.getSamzaSqlRelRecord().getFieldValues().get(idx)); } return new SamzaSqlCompositeKey(keyParts); } }
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex, MessageStream<SamzaSqlRelMessage> inputStream) { return inputStream.flatMap(message -> { Object field = message.getSamzaSqlRelRecord().getFieldValues().get(flattenIndex); if (field != null && field instanceof List) { List<SamzaSqlRelMessage> outMessages = new ArrayList<>(); for (Object fieldValue : (List) field) { List<Object> newValues = new ArrayList<>(message.getSamzaSqlRelRecord().getFieldValues()); newValues.set(flattenIndex, Collections.singletonList(fieldValue)); outMessages.add(new SamzaSqlRelMessage(message.getSamzaSqlRelRecord().getFieldNames(), newValues)); } return outMessages; } else { return Collections.singletonList(message); } }); }
@Override protected List<Object> getTableRelRecordFieldValues(KV record) { // Using the message rel converter, convert message to sql rel message and add to output values. SamzaSqlRelMessage relMessage = msgConverter.convertToRelMessage(record); return relMessage.getSamzaSqlRelRecord().getFieldValues(); }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values from. * @param keyValueIdx list of key values in the form of field indices within the rel message. * @param keyPartNames Represents the key field names. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> keyValueIdx, List<String> keyPartNames) { Validate.isTrue(keyValueIdx.size() == keyPartNames.size(), "Key part name and value list sizes are different"); ArrayList<Object> keyPartValues = new ArrayList<>(); for (int idx : keyValueIdx) { keyPartValues.add(message.getSamzaSqlRelRecord().getFieldValues().get(idx)); } return new SamzaSqlRelRecord(keyPartNames, keyPartValues); }
@Override public boolean apply(SamzaSqlRelMessage message) { Object[] result = new Object[1]; expr.execute(context.getExecutionContext(), context.getDataContext(), message.getSamzaSqlRelRecord().getFieldValues().toArray(), result); if (result.length > 0 && result[0] instanceof Boolean) { boolean retVal = (Boolean) result[0]; log.debug( String.format("return value for input %s is %s", Arrays.asList(message.getSamzaSqlRelRecord().getFieldValues()).toString(), retVal)); return retVal; } else { log.error("return value is not boolean"); return false; } } }
@Override protected List<Object> getTableRelRecordFieldValues(KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record) { return record.getValue().getSamzaSqlRelRecord().getFieldValues(); }
@Override public boolean apply(SamzaSqlRelMessage message) { Instant startProcessing = Instant.now(); Object[] result = new Object[1]; expr.execute(translatorContext.getExecutionContext(), translatorContext.getDataContext(), message.getSamzaSqlRelRecord().getFieldValues().toArray(), result); if (result.length > 0 && result[0] instanceof Boolean) { boolean retVal = (Boolean) result[0]; log.debug( String.format("return value for input %s is %s", Arrays.asList(message.getSamzaSqlRelRecord().getFieldValues()).toString(), retVal)); updateMetrics(startProcessing, retVal, Instant.now()); return retVal; } else { log.error("return value is not boolean"); return false; } }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values and names from. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { return createSamzaSqlCompositeKey(message, relIdx, getSamzaSqlCompositeKeyFieldNames(message.getSamzaSqlRelRecord().getFieldNames(), relIdx)); }
protected KV<Object, Object> convertToSamzaMessage(SamzaSqlRelMessage relMessage, Schema payloadSchema) { return new KV<>(relMessage.getKey(), convertToGenericRecord(relMessage.getSamzaSqlRelRecord(), payloadSchema)); }
@Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message) { RelDataType type = project.getRowType(); Object[] output = new Object[type.getFieldCount()]; expr.execute(context.getExecutionContext(), context.getDataContext(), message.getSamzaSqlRelRecord().getFieldValues().toArray(), output); List<String> names = new ArrayList<>(); for (int index = 0; index < output.length; index++) { names.add(index, project.getNamedProjects().get(index).getValue()); } return new SamzaSqlRelMessage(names, Arrays.asList(output)); } }
protected KV<Object, Object> convertToSamzaMessage(SamzaSqlRelMessage relMessage, Schema avroSchema) { return new KV<>(relMessage.getKey(), convertToGenericRecord(relMessage.getSamzaSqlRelRecord(), avroSchema)); }
/** * transforms the input message into the output message with projected fields * @param message the input message to be transformed * @return the new SamzaSqlRelMessage message */ @Override public SamzaSqlRelMessage apply(SamzaSqlRelMessage message) { Instant arrivalTime = Instant.now(); RelDataType type = project.getRowType(); Object[] output = new Object[type.getFieldCount()]; expr.execute(translatorContext.getExecutionContext(), translatorContext.getDataContext(), message.getSamzaSqlRelRecord().getFieldValues().toArray(), output); List<String> names = new ArrayList<>(); for (int index = 0; index < output.length; index++) { names.add(index, project.getNamedProjects().get(index).getValue()); } updateMetrics(arrivalTime, Instant.now(), message.getSamzaSqlRelMsgMetadata().isNewInputMessage); return new SamzaSqlRelMessage(names, Arrays.asList(output), message.getSamzaSqlRelMsgMetadata()); }
@Test public void testGetField() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertEquals(values.get(0), message.getSamzaSqlRelRecord().getField(names.get(0)).get()); Assert.assertEquals(values.get(1), message.getSamzaSqlRelRecord().getField(names.get(1)).get()); }
@Test public void testEmptyRecordConversion() { GenericData.Record record = new GenericData.Record(SimpleRecord.SCHEMA$); SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", record)); Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), message.getSamzaSqlRelRecord().getFieldValues().size()); }
@Test public void testNullRecordConversion() { SamzaSqlRelMessage message = simpleRecordAvroRelConverter.convertToRelMessage(new KV<>("key", null)); Assert.assertEquals(message.getSamzaSqlRelRecord().getFieldNames().size(), message.getSamzaSqlRelRecord().getFieldValues().size()); }
SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, null); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(streamFieldNames); expectedFieldNames.addAll(tableFieldNames); List<Object> expectedFieldValues = new ArrayList<>(streamFieldValues); expectedFieldValues.addAll(tableFieldNames.stream().map( name -> null ).collect(Collectors.toList())); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues);
@Test public void testGetNonExistentField() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertFalse(message.getSamzaSqlRelRecord().getField("field3").isPresent()); }
@Test public void testWithInnerJoinWithTableOnRight() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = new SamzaSqlRelMessage(tableFieldNames, tableFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 1); List<Integer> tableKeyIds = Arrays.asList(0, 1); SamzaSqlRelRecord compositeKey = SamzaSqlRelMessage.createSamzaSqlCompositeKey(tableMsg, tableKeyIds); KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record = KV.of(compositeKey, tableMsg); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(true); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(false); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(streamFieldNames); expectedFieldNames.addAll(tableFieldNames); List<Object> expectedFieldValues = new ArrayList<>(streamFieldValues); expectedFieldValues.addAll(tableFieldValues); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues); }
@Test public void testWithDifferentFields() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessageSerde serde = (SamzaSqlRelMessageSerde) new SamzaSqlRelMessageSerdeFactory().getSerde(null, null); SamzaSqlRelMessage resultMsg = serde.fromBytes(serde.toBytes(message)); Assert.assertEquals(names, resultMsg.getSamzaSqlRelRecord().getFieldNames()); Assert.assertEquals(values, resultMsg.getSamzaSqlRelRecord().getFieldValues()); }
@Test public void testWithDifferentFields() { SamzaSqlRelRecord record = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")).getSamzaSqlRelRecord(); SamzaSqlRelRecordSerde serde = (SamzaSqlRelRecordSerde) new SamzaSqlRelRecordSerdeFactory().getSerde(null, null); SamzaSqlRelRecord resultRecord = serde.fromBytes(serde.toBytes(record)); Assert.assertEquals(names, resultRecord.getFieldNames()); Assert.assertEquals(values, resultRecord.getFieldValues()); }