/** * Converts the nested avro object in SamzaMessage to relational message corresponding to * the tableName with relational schema. */ @Override public SamzaSqlRelMessage convertToRelMessage(KV<Object, Object> samzaMessage) { List<String> payloadFieldNames = new ArrayList<>(); List<Object> payloadFieldValues = new ArrayList<>(); Object value = samzaMessage.getValue(); if (value instanceof IndexedRecord) { fetchFieldNamesAndValuesFromIndexedRecord((IndexedRecord) value, payloadFieldNames, payloadFieldValues, payloadSchema); } else if (value == null) { // If the payload is null, set each record value as null. payloadFieldNames.addAll(payloadSchema.getFields().stream().map(Schema.Field::name).collect(Collectors.toList())); IntStream.range(0, payloadFieldNames.size()).forEach(x -> payloadFieldValues.add(null)); } else { String msg = "Avro message converter doesn't support messages of type " + value.getClass(); LOG.error(msg); throw new SamzaException(msg); } return new SamzaSqlRelMessage(samzaMessage.getKey(), payloadFieldNames, payloadFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); }
@Override public Object apply(IncomingMessageEnvelope ime) { Assert.notNull(ime, "ime is null"); KV<Object, Object> keyAndMessageKV = KV.of(ime.getKey(), ime.getMessage()); SamzaSqlRelMsgMetadata metadata = new SamzaSqlRelMsgMetadata(Instant.ofEpochMilli(ime.getEventTime()).toString(), Instant.ofEpochMilli(ime.getArrivalTime()).toString(), null); SamzaSqlInputMessage samzaMsg = SamzaSqlInputMessage.of(keyAndMessageKV, metadata); return samzaMsg; } }
@Test public void testInEquality() { SamzaSqlRelMessage message1 = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage message2 = new SamzaSqlRelMessage(Arrays.asList("field1", "field2"), Arrays.asList("value2", "value2"), new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertNotEquals(message1, message2); Assert.assertNotEquals(message1.hashCode(), message2.hashCode()); }
@Test public void testEquality() { SamzaSqlRelMessage message1 = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage message2 = new SamzaSqlRelMessage(Arrays.asList("field1", "field2"), Arrays.asList("value1", "value2"), new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertEquals(message1, message2); Assert.assertEquals(message1.hashCode(), message2.hashCode()); }
@Test (expected = IllegalArgumentException.class) public void testCompositeKeyCreationWithInEqualKeyNameValues() { List<String> keyPartNames = Arrays.asList("kfield1", "kfield2"); SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1))); } }
private MessageStream<SamzaSqlRelMessage> translateFlatten(Integer flattenIndex, MessageStream<SamzaSqlRelMessage> inputStream) { return inputStream.flatMap(message -> { Object field = message.getSamzaSqlRelRecord().getFieldValues().get(flattenIndex); if (field != null && field instanceof List) { List<SamzaSqlRelMessage> outMessages = new ArrayList<>(); SamzaSqlRelMsgMetadata messageMetadata = message.getSamzaSqlRelMsgMetadata(); SamzaSqlRelMsgMetadata newMetadata = new SamzaSqlRelMsgMetadata(messageMetadata.getEventTime(), messageMetadata.getarrivalTime(), messageMetadata.getscanTime(), true); for (Object fieldValue : (List) field) { List<Object> newValues = new ArrayList<>(message.getSamzaSqlRelRecord().getFieldValues()); newValues.set(flattenIndex, Collections.singletonList(fieldValue)); outMessages.add(new SamzaSqlRelMessage(message.getSamzaSqlRelRecord().getFieldNames(), newValues, newMetadata)); newMetadata = new SamzaSqlRelMsgMetadata(newMetadata.getEventTime(), newMetadata.getarrivalTime(), newMetadata.getscanTime(), false); } return outMessages; } else { message.getSamzaSqlRelMsgMetadata().isNewInputMessage = true; return Collections.singletonList(message); } }); }
@Test public void testGetField() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertEquals(values.get(0), message.getSamzaSqlRelRecord().getField(names.get(0)).get()); Assert.assertEquals(values.get(1), message.getSamzaSqlRelRecord().getField(names.get(1)).get()); }
@Test public void testGetNonExistentField() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); Assert.assertFalse(message.getSamzaSqlRelRecord().getField("field3").isPresent()); }
@Test public void testCompositeKeyCreation() { List<String> keyPartNames = Arrays.asList("kfield1", "kfield2"); SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelRecord relRecord1 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Collections.singletonList(0)); Assert.assertEquals(relRecord1.getFieldNames().size(), 1); Assert.assertEquals(relRecord1.getFieldNames().get(0), "field1"); Assert.assertEquals(relRecord1.getFieldValues().get(0), "value1"); SamzaSqlRelRecord relRecord2 = SamzaSqlRelMessage.createSamzaSqlCompositeKey(message, Arrays.asList(1, 0), SamzaSqlRelMessage.getSamzaSqlCompositeKeyFieldNames(keyPartNames, Arrays.asList(1, 0))); Assert.assertEquals(relRecord2.getFieldNames().size(), 2); Assert.assertEquals(relRecord2.getFieldNames().get(0), "kfield2"); Assert.assertEquals(relRecord2.getFieldValues().get(0), "value2"); Assert.assertEquals(relRecord2.getFieldNames().get(1), "kfield1"); Assert.assertEquals(relRecord2.getFieldValues().get(1), "value1"); }
@Test public void testWithInnerJoinWithTableOnRight() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = new SamzaSqlRelMessage(tableFieldNames, tableFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 1); List<Integer> tableKeyIds = Arrays.asList(0, 1); SamzaSqlRelRecord compositeKey = SamzaSqlRelMessage.createSamzaSqlCompositeKey(tableMsg, tableKeyIds); KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record = KV.of(compositeKey, tableMsg); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(true); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(false); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(streamFieldNames); expectedFieldNames.addAll(tableFieldNames); List<Object> expectedFieldValues = new ArrayList<>(streamFieldValues); expectedFieldValues.addAll(tableFieldValues); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues); }
@Test public void testNullRecordWithLeftOuterJoin() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.LEFT; List<Integer> streamKeyIds = Arrays.asList(0, 1);
@Test public void testNullRecordWithInnerJoin() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 1); List<Integer> tableKeyIds = Arrays.asList(2, 3); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(true); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(false); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, null); Assert.assertNull(outMsg); }
@Test public void testWithInnerJoinWithTableOnLeft() { SamzaSqlRelMessage streamMsg = new SamzaSqlRelMessage(streamFieldNames, streamFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = new SamzaSqlRelMessage(tableFieldNames, tableFieldValues, new SamzaSqlRelMsgMetadata("", "", "")); JoinRelType joinRelType = JoinRelType.INNER; List<Integer> streamKeyIds = Arrays.asList(0, 2); List<Integer> tableKeyIds = Arrays.asList(0, 2); SamzaSqlRelRecord compositeKey = SamzaSqlRelMessage.createSamzaSqlCompositeKey(tableMsg, tableKeyIds); KV<SamzaSqlRelRecord, SamzaSqlRelMessage> record = KV.of(compositeKey, tableMsg); JoinInputNode mockTableInputNode = mock(JoinInputNode.class); when(mockTableInputNode.getKeyIds()).thenReturn(tableKeyIds); when(mockTableInputNode.isPosOnRight()).thenReturn(false); when(mockTableInputNode.getFieldNames()).thenReturn(tableFieldNames); JoinInputNode mockStreamInputNode = mock(JoinInputNode.class); when(mockStreamInputNode.getKeyIds()).thenReturn(streamKeyIds); when(mockStreamInputNode.isPosOnRight()).thenReturn(true); when(mockStreamInputNode.getFieldNames()).thenReturn(streamFieldNames); SamzaSqlLocalTableJoinFunction joinFn = new SamzaSqlLocalTableJoinFunction(mockStreamInputNode, mockTableInputNode, joinRelType); SamzaSqlRelMessage outMsg = joinFn.apply(streamMsg, record); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues().size(), outMsg.getSamzaSqlRelRecord().getFieldNames().size()); List<String> expectedFieldNames = new ArrayList<>(tableFieldNames); expectedFieldNames.addAll(streamFieldNames); List<Object> expectedFieldValues = new ArrayList<>(tableFieldValues); expectedFieldValues.addAll(streamFieldValues); Assert.assertEquals(outMsg.getSamzaSqlRelRecord().getFieldValues(), expectedFieldValues); }
@Test public void testWithDifferentFields() { SamzaSqlRelRecord record = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")).getSamzaSqlRelRecord(); SamzaSqlRelRecordSerde serde = (SamzaSqlRelRecordSerde) new SamzaSqlRelRecordSerdeFactory().getSerde(null, null); SamzaSqlRelRecord resultRecord = serde.fromBytes(serde.toBytes(record)); Assert.assertEquals(names, resultRecord.getFieldNames()); Assert.assertEquals(values, resultRecord.getFieldValues()); }
void translate(final LogicalAggregate aggregate, final TranslatorContext context) { validateAggregateFunctions(aggregate); MessageStream<SamzaSqlRelMessage> inputStream = context.getMessageStream(aggregate.getInput().getId()); // At this point, the assumption is that only count function is supported. SupplierFunction<Long> initialValue = () -> (long) 0; FoldLeftFunction<SamzaSqlRelMessage, Long> foldCountFn = (m, c) -> c + 1; final ArrayList<String> aggFieldNames = getAggFieldNames(aggregate); MessageStream<SamzaSqlRelMessage> outputStream = inputStream .map(new TranslatorInputMetricsMapFunction(logicalOpId)) .window(Windows.keyedTumblingWindow(m -> m, Duration.ofMillis(context.getExecutionContext().getSamzaSqlApplicationConfig().getWindowDurationMs()), initialValue, foldCountFn, new SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde(), new LongSerde()) .setAccumulationMode( AccumulationMode.DISCARDING), changeLogStorePrefix + "_tumblingWindow_" + logicalOpId) .map(windowPane -> { List<String> fieldNames = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldNames(); List<Object> fieldValues = windowPane.getKey().getKey().getSamzaSqlRelRecord().getFieldValues(); fieldNames.add(aggFieldNames.get(0)); fieldValues.add(windowPane.getMessage()); return new SamzaSqlRelMessage(fieldNames, fieldValues, new SamzaSqlRelMsgMetadata("", "", "")); }); context.registerMessageStream(aggregate.getId(), outputStream); outputStream.map(new TranslatorOutputMetricsMapFunction(logicalOpId)); }
new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlExecutionContext executionContext = mock(SamzaSqlExecutionContext.class); DataContext dataContext = mock(DataContext.class);
}}, new ArrayList<Object>() {{ this.add(testObj); }}, new SamzaSqlRelMsgMetadata("", "", "")); Collection<SamzaSqlRelMessage> flattenedMsgs = flattenOp.getTransformFn().apply(mockMsg); assertTrue(flattenedMsgs.size() == 1); }}, new ArrayList<Object>() {{ this.add(testList); }}, new SamzaSqlRelMsgMetadata("", "", "")); flattenedMsgs = flattenOp.getTransformFn().apply(mockMsg); assertTrue(flattenedMsgs.size() == 10); new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlExecutionContext executionContext = mock(SamzaSqlExecutionContext.class); DataContext dataContext = mock(DataContext.class);
@Test public void testWithDifferentFields() { SamzaSqlRelMessage message = new SamzaSqlRelMessage(names, values, new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessageSerde serde = (SamzaSqlRelMessageSerde) new SamzaSqlRelMessageSerdeFactory().getSerde(null, null); SamzaSqlRelMessage resultMsg = serde.fromBytes(serde.toBytes(message)); Assert.assertEquals(names, resultMsg.getSamzaSqlRelRecord().getFieldNames()); Assert.assertEquals(values, resultMsg.getSamzaSqlRelRecord().getFieldValues()); }
new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlExecutionContext executionContext = mock(SamzaSqlExecutionContext.class); DataContext dataContext = mock(DataContext.class);
new SamzaSqlRelMsgMetadata("", "", "")); SamzaSqlRelMessage tableMsg = relConverter.convertToRelMessage(new KV(tableRecord.get("id"), tableRecord)); JoinRelType joinRelType = JoinRelType.INNER;