public static SamzaSqlRelRecord getMessageKeyRelRecord(SamzaSqlRelMessage message, List<Integer> streamFieldIds, List<String> tableFieldNames, List<Integer> tableKeyIds) { return createSamzaSqlCompositeKey(message, streamFieldIds, getSamzaSqlCompositeKeyFieldNames(tableFieldNames, tableKeyIds)); }
/** * Create composite key from the rel message. * @param message Represents the samza sql rel message to extract the key values and names from. * @param relIdx list of keys in the form of field indices within the rel message. * @return the composite key of the rel message */ public static SamzaSqlRelRecord createSamzaSqlCompositeKey(SamzaSqlRelMessage message, List<Integer> relIdx) { return createSamzaSqlCompositeKey(message, relIdx, getSamzaSqlCompositeKeyFieldNames(message.getSamzaSqlRelRecord().getFieldNames(), relIdx)); }
@Test (expected = IllegalArgumentException.class) public void testCompositeKeyCreationWithInEqualKeyNameValues() { List<String> keyPartNames = Arrays.asList("kfield1", "kfield2"); SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1))); } }
inputStream .map(inputMetricsMF) .partitionBy(m -> createSamzaSqlCompositeKey(m, streamKeyIds, getSamzaSqlCompositeKeyFieldNames(tableFieldNames, tableKeyIds)), m -> m, KVSerde.of(keySerde, valueSerde), intermediateStreamPrefix + "stream_" + logicalOpId)
@Test public void testCompositeKeyCreation() { List<String> keyPartNames = Arrays.asList("kfield1", "kfield2"); SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Collections.singletonList(0)); Assert.assertEquals(relRecord1.getFieldNames().size(), 1); Assert.assertEquals(relRecord1.getFieldNames().get(0), "field1"); Assert.assertEquals(relRecord1.getFieldValues().get(0), "value1"); SamzaSqlRelRecord relRecord2 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1, 0))); Assert.assertEquals(relRecord2.getFieldNames().size(), 2); Assert.assertEquals(relRecord2.getFieldNames().get(0), "kfield2"); Assert.assertEquals(relRecord2.getFieldValues().get(0), "value2"); Assert.assertEquals(relRecord2.getFieldNames().get(1), "kfield1"); Assert.assertEquals(relRecord2.getFieldValues().get(1), "value1"); }
.partitionBy(m -> createSamzaSqlCompositeKey(m, tableKeyIds), m -> m, KVSerde.of(keySerde, valueSerde), intermediateStreamPrefix + "table_" + logicalOpId) .sendTo(table);
@Test public void testWithInnerJoinWithTableOnRight() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = new SamzaSqlRelMessage(tableFieldNames, tableFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 1); List<Integer> tableKeyIds = Arrays.asList(0, 1); SamzaSqlRelRecord compositeKey = SamzaSqlRelMessage.createSamzaSqlCompositeKey(tableMsg, tableKeyIds); KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record = KV.of(compositeKey, tableMsg); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(true); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(false); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(streamFieldNames); expectedFieldNames.addAll(tableFieldNames); List<Object> expectedFieldValues = new ArrayList<>(streamFieldValues); expectedFieldValues.addAll(tableFieldValues); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues); }
@Test public void testWithInnerJoinWithTableOnLeft() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = new SamzaSqlRelMessage(tableFieldNames, tableFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 2); List<Integer> tableKeyIds = Arrays.asList(0, 2); SamzaSqlRelRecord compositeKey = SamzaSqlRelMessage.createSamzaSqlCompositeKey(tableMsg, tableKeyIds); KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record = KV.of(compositeKey, tableMsg); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(false); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(true); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(tableFieldNames); expectedFieldNames.addAll(streamFieldNames); List<Object> expectedFieldValues = new ArrayList<>(tableFieldValues); expectedFieldValues.addAll(streamFieldValues); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues); }