private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamStreamJoin() { /** * Creates the following stream-stream join which is invalid due to partition count disagreement * between the 2 input streams. * * input1 (64) -- * | * join -> output1 (8) * | * input3 (32) -- */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); messageStream1 .join(messageStream3, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(2), "j1") .sendTo(output1); }, config); }
private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) { SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream); MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId()); MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId)); Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor(); if (!tableDescriptor.isPresent()) { KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); String systemName = sinkConfig.getSystemName(); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde); OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd)); outputStream.sendTo(stm); } else { Table outputTable = appDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource(); throw new SamzaException(msg); } outputStream.sendTo(outputTable); } } }
try { List<String> dslStmts = SamzaSqlDslConverter.fetchSqlFromConfig(appDescriptor.getConfig()); SamzaSqlApplicationConfig.populateSystemStreamsAndGetRelRoots(dslStmts, appDescriptor.getConfig(), inputSystemStreams, outputSystemStreams); new SamzaSqlApplicationConfig(appDescriptor.getConfig(), inputSystemStreams, outputSystemStreams); appDescriptor.withApplicationTaskContextFactory(new ApplicationTaskContextFactory<SamzaSqlApplicationContext>() { @Override public SamzaSqlApplicationContext create(ExternalContext externalContext, JobContext jobContext,
@Test(expected = IllegalStateException.class) public void testSetDefaultSystemDescriptorAfterGettingOutputStream() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd); appDesc.withDefaultSystem(sd); // should throw exception }, getConfig()); }
public ExampleExpandingSystemDescriptor(String systemName) { super(systemName, FACTORY_CLASS_NAME, (InputTransformer<String>) IncomingMessageEnvelope::toString, (streamGraph, inputDescriptor) -> (MessageStream<Long>) streamGraph.getInputStream(inputDescriptor) ); }
MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); MessageStream<KV<Object, Object>> messageStream3 = appDesc.getInputStream(input3Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); Table table = appDesc.getTable(tableDescriptor);
@Test public void testMultipleSystemDescriptorForSameSystemName() { GenericSystemDescriptor sd1 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericSystemDescriptor sd2 = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd1.getInputDescriptor("test-stream-1", mock(Serde.class)); GenericInputDescriptor isd2 = sd2.getInputDescriptor("test-stream-2", mock(Serde.class)); GenericOutputDescriptor osd1 = sd2.getOutputDescriptor("test-stream-3", mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); try { appDesc.getInputStream(isd2); fail("Adding input stream with the same system name but different SystemDescriptor should have failed"); } catch (IllegalStateException e) { } try { appDesc.getOutputStream(osd1); fail("adding output stream with the same system name but different SystemDescriptor should have failed"); } catch (IllegalStateException e) { } }, getConfig()); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.withDefaultSystem(sd2); try { appDesc.getInputStream(isd1); fail("Adding input stream with the same system name as the default system but different SystemDescriptor should have failed"); } catch (IllegalStateException e) { } }, getConfig()); }
private StreamApplicationDescriptorImpl createStreamGraphWithStreamTableJoinAndSendToSameTable() { /** * A special example of stream-table join where a stream is joined with a table, and the result is * sent to the same table. This example is necessary to ensure {@link ExecutionPlanner} does not * get stuck traversing the virtual cycle between stream-table-join and send-to-table operator specs * indefinitely. * * The reason such virtual cycle is present is to support computing partitions of intermediate * streams participating in stream-table joins. Please, refer to SAMZA SEP-16 for more details. */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .sendTo(table); }, config); }
@Test(expected = IllegalStateException.class) public void testSetDefaultSystemDescriptorAfterGettingInputStream() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); appDesc.withDefaultSystem(sd); // should throw exception }, getConfig()); }
@Test public void testGetTable() throws Exception { Config mockConfig = getConfig(); String tableId = "t1"; BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class); when(mockTableDescriptor.getTableId()).thenReturn(tableId); AtomicReference<TableImpl> table = new AtomicReference<>(); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { table.set((TableImpl) appDesc.getTable(mockTableDescriptor)); }, mockConfig); assertEquals(tableId, table.get().getTableId()); }
@Test(expected = IllegalStateException.class) public void testGetSameOutputStreamTwice() { String streamId = "test-stream-1"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd1 = sd.getOutputDescriptor(streamId, mock(Serde.class)); GenericOutputDescriptor osd2 = sd.getOutputDescriptor(streamId, mock(Serde.class)); new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd1); appDesc.getOutputStream(osd2); // should throw exception }, getConfig()); }
/** * For unit testing only */ @VisibleForTesting void translate(SamzaSqlQueryParser.QueryInfo queryInfo, StreamApplicationDescriptor appDesc, int queryId) { QueryPlanner planner = new QueryPlanner(sqlConfig.getRelSchemaProviders(), sqlConfig.getInputSystemStreamConfigBySource(), sqlConfig.getUdfMetadata()); final RelRoot relRoot = planner.plan(queryInfo.getSelectQuery()); SamzaSqlExecutionContext executionContext = new SamzaSqlExecutionContext(sqlConfig); TranslatorContext translatorContext = new TranslatorContext(appDesc, relRoot, executionContext); translate(relRoot, sqlConfig.getOutputSystemStreams().get(queryId), translatorContext, queryId); Map<Integer, TranslatorContext> translatorContexts = new HashMap<>(); translatorContexts.put(queryId, translatorContext.clone()); appDesc.withApplicationTaskContextFactory(new ApplicationTaskContextFactory<SamzaSqlApplicationContext>() { @Override public SamzaSqlApplicationContext create(ExternalContext externalContext, JobContext jobContext, ContainerContext containerContext, TaskContext taskContext, ApplicationContainerContext applicationContainerContext) { return new SamzaSqlApplicationContext(translatorContexts); } }); }
@Test public void testApplicationContainerContextFactory() { ApplicationContainerContextFactory factory = mock(ApplicationContainerContextFactory.class); StreamApplication testApp = appDesc -> appDesc.withApplicationContainerContextFactory(factory); StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appSpec.getApplicationContainerContextFactory(), Optional.of(factory)); }
StreamApplication getRepartitionOnlyStreamApplication() { return appDesc -> { MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor); input1.partitionBy(KV::getKey, KV::getValue, mock(KVSerde.class), "p1"); }; }
private StreamApplicationDescriptorImpl createStreamGraphWithInvalidStreamTableJoin() { /** * Example stream-table join that is invalid due to disagreement in partition count * between the 2 input streams. * * input1 (64) -> send-to-table t * * join-table t -> output1 (8) * | * input2 (16) ————————— * */ return new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> messageStream1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<Object, Object>> messageStream2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); TableDescriptor tableDescriptor = new TestLocalTableDescriptor.MockLocalTableDescriptor( "table-id", new KVSerde(new StringSerde(), new StringSerde())); Table table = appDesc.getTable(tableDescriptor); messageStream1.sendTo(table); messageStream1 .join(table, mock(StreamTableJoinFunction.class)) .join(messageStream2, mock(JoinFunction.class), mock(Serde.class), mock(Serde.class), mock(Serde.class), Duration.ofHours(1), "j2") .sendTo(output1); }, config); }
@Test(expected = IllegalStateException.class) public void testGetTableWithBadId() { Config mockConfig = getConfig(); new StreamApplicationDescriptorImpl(appDesc -> { BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class); when(mockTableDescriptor.getTableId()).thenReturn("my.table"); appDesc.getTable(mockTableDescriptor); }, mockConfig); }
@Test public void testGetOutputStreamWithKeyValueSerde() { String streamId = "test-stream-1"; KVSerde mockKVSerde = mock(KVSerde.class); Serde mockKeySerde = mock(Serde.class); Serde mockValueSerde = mock(Serde.class); doReturn(mockKeySerde).when(mockKVSerde).getKeySerde(); doReturn(mockValueSerde).when(mockKVSerde).getValueSerde(); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockKVSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd); }, getConfig()); OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId); assertEquals(streamId, outputStreamImpl.getStreamId()); assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId)); assertEquals(mockKeySerde, outputStreamImpl.getKeySerde()); assertEquals(mockValueSerde, outputStreamImpl.getValueSerde()); }
@Test public void testApplicationTaskContextFactory() { ApplicationTaskContextFactory factory = mock(ApplicationTaskContextFactory.class); StreamApplication testApp = appDesc -> appDesc.withApplicationTaskContextFactory(factory); StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.of(factory)); }
private StreamApplicationDescriptorImpl createSimpleGraph() { /** * a simple graph of partitionBy and map * * input1 -> partitionBy -> map -> output1 * */ return new StreamApplicationDescriptorImpl(appDesc-> { MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input1Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); input1 .partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1") .map(kv -> kv) .sendTo(output1); }, config); }
StreamApplication getBroadcastOnlyStreamApplication(Serde serde) { return appDesc -> { MessageStream<KV<String, Object>> input = appDesc.getInputStream(input1Descriptor); input.broadcast(serde, "b1"); }; } }