@Test(expected = IllegalStateException.class) public void testSetDefaultSystemDescriptorAfterGettingIntermediateStream() { String streamId = "test-stream-1"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false); streamAppDesc.withDefaultSystem(mock(SystemDescriptor.class)); // should throw exception }
/** * Creates a {@link TaskFactory} based on {@link ApplicationDescriptorImpl} * * @param appDesc {@link ApplicationDescriptorImpl} for this application * @return {@link TaskFactory} object defined by {@code appDesc} */ public static TaskFactory getTaskFactory(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { if (appDesc instanceof TaskApplicationDescriptorImpl) { return ((TaskApplicationDescriptorImpl) appDesc).getTaskFactory(); } else if (appDesc instanceof StreamApplicationDescriptorImpl) { return (StreamTaskFactory) () -> new StreamOperatorTask( ((StreamApplicationDescriptorImpl) appDesc).getOperatorSpecGraph()); } throw new IllegalArgumentException(String.format("ApplicationDescriptorImpl has to be either TaskApplicationDescriptorImpl or " + "StreamApplicationDescriptorImpl. class %s is not supported", appDesc.getClass().getName())); }
/** * Gets the unique ID for the next operator in the graph. The ID is of the following format: * jobName-jobId-opCode-(userDefinedId|nextOpNum); * * @param opCode the {@link OpCode} of the next operator * @param userDefinedId the optional user-provided name of the next operator or null * @return the unique ID for the next operator in the graph */ public String getNextOpId(OpCode opCode, String userDefinedId) { if (StringUtils.isNotBlank(userDefinedId) && !ID_PATTERN.matcher(userDefinedId).matches()) { throw new SamzaException("Operator ID must not contain spaces or special characters: " + userDefinedId); } JobConfig jobConfig = new JobConfig(getConfig()); String nextOpId = String.format("%s-%s-%s-%s", jobConfig.getName().get(), jobConfig.getJobId(), opCode.name().toLowerCase(), StringUtils.isNotBlank(userDefinedId) ? userDefinedId.trim() : String.valueOf(nextOpNum)); if (!operatorIds.add(nextOpId)) { throw new SamzaException( String.format("Found duplicate operator ID %s in the graph. Operator IDs must be unique.", nextOpId)); } nextOpNum++; return nextOpId; }
public OperatorSpecGraph(StreamApplicationDescriptorImpl streamAppDesc) { this.inputOperators = streamAppDesc.getInputOperators(); this.outputStreams = streamAppDesc.getOutputStreams(); this.intermediateBroadcastStreamIds = streamAppDesc.getIntermediateBroadcastStreamIds(); this.allOpSpecs = Collections.unmodifiableSet(this.findAllOperatorSpecs()); this.hasWindowOrJoins = checkWindowOrJoins(); this.serializedOpSpecGraph = opSpecGraphSerde.toBytes(this); }
@Override public MessageStream<M> broadcast(Serde<M> serde, String userDefinedId) { String opId = this.streamAppDesc.getNextOpId(OpCode.BROADCAST, userDefinedId); IntermediateMessageStreamImpl<M> intermediateStream = this.streamAppDesc.getIntermediateStream(opId, serde, true); BroadcastOperatorSpec<M> broadcastOperatorSpec = OperatorSpecs.createBroadCastOperatorSpec(intermediateStream.getOutputStream(), opId); this.operatorSpec.registerNextOperatorSpec(broadcastOperatorSpec); return intermediateStream; }
@Override public <K, V> Table<KV<K, V>> getTable(TableDescriptor<K, V, ?> tableDescriptor) { addTableDescriptor(tableDescriptor); if (tableDescriptor instanceof LocalTableDescriptor) { LocalTableDescriptor localTableDescriptor = (LocalTableDescriptor) tableDescriptor; getOrCreateTableSerdes(localTableDescriptor.getTableId(), localTableDescriptor.getSerde()); } return new TableImpl(tableDescriptor); }
@Test public void testGetIntermediateStreamWithDefaultSystemDescriptor() { Config mockConfig = getConfig(); String streamId = "streamId"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, mockConfig); GenericSystemDescriptor sd = new GenericSystemDescriptor("mock-system", "mock-system-factory"); streamAppDesc.withDefaultSystem(sd); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mock(Serde.class), false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
@Test public void testEmptyChain() { StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> { }, mock(Config.class)); OperatorImplGraph opGraph = new OperatorImplGraph(graphSpec.getOperatorSpecGraph(), context, mock(Clock.class)); assertEquals(0, opGraph.getAllInputOperators().size()); }
@Test public void testMultipleGetInputStreams() { String streamId1 = "test-stream-1"; String streamId2 = "test-stream-2"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd1 = sd.getInputDescriptor(streamId1, mock(Serde.class)); GenericInputDescriptor isd2 = sd.getInputDescriptor(streamId2, mock(Serde.class)); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd1); appDesc.getInputStream(isd2); }, getConfig()); InputOperatorSpec inputOpSpec1 = streamAppDesc.getInputOperators().get(streamId1); InputOperatorSpec inputOpSpec2 = streamAppDesc.getInputOperators().get(streamId2); assertEquals(2, streamAppDesc.getInputOperators().size()); assertEquals(streamId1, inputOpSpec1.getStreamId()); assertEquals(streamId2, inputOpSpec2.getStreamId()); assertEquals(2, streamAppDesc.getInputDescriptors().size()); assertEquals(isd1, streamAppDesc.getInputDescriptors().get(streamId1)); assertEquals(isd2, streamAppDesc.getInputDescriptors().get(streamId2)); }
@Test public void testGetInputStreamPreservesInsertionOrder() { Config mockConfig = getConfig(); String testStreamId1 = "test-stream-1"; String testStreamId2 = "test-stream-2"; String testStreamId3 = "test-stream-3"; GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(sd.getInputDescriptor(testStreamId1, mock(Serde.class))); appDesc.getInputStream(sd.getInputDescriptor(testStreamId2, mock(Serde.class))); appDesc.getInputStream(sd.getInputDescriptor(testStreamId3, mock(Serde.class))); }, mockConfig); List<InputOperatorSpec> inputSpecs = new ArrayList<>(streamAppDesc.getInputOperators().values()); assertEquals(inputSpecs.size(), 3); assertEquals(inputSpecs.get(0).getStreamId(), testStreamId1); assertEquals(inputSpecs.get(1).getStreamId(), testStreamId2); assertEquals(inputSpecs.get(2).getStreamId(), testStreamId3); }
@Override public <M> OutputStream<M> getOutputStream(OutputDescriptor<M, ?> outputDescriptor) { addOutputDescriptor(outputDescriptor); String streamId = outputDescriptor.getStreamId(); Serde serde = outputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; outputStreams.put(streamId, new OutputStreamImpl<>(streamId, kvSerdes.getKey(), kvSerdes.getValue(), isKeyed)); return outputStreams.get(streamId); }
/** * Gets the unique ID for the next operator in the graph. The ID is of the following format: * jobName-jobId-opCode-nextOpNum; * * @param opCode the {@link OpCode} of the next operator * @return the unique ID for the next operator in the graph */ public String getNextOpId(OpCode opCode) { return getNextOpId(opCode, null); }
@Test public void testGetTable() throws Exception { Config mockConfig = getConfig(); String tableId = "t1"; BaseTableDescriptor mockTableDescriptor = mock(BaseTableDescriptor.class); when(mockTableDescriptor.getTableId()).thenReturn(tableId); AtomicReference<TableImpl> table = new AtomicReference<>(); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { table.set((TableImpl) appDesc.getTable(mockTableDescriptor)); }, mockConfig); assertEquals(tableId, table.get().getTableId()); }
@Test(expected = IllegalStateException.class) public void testGetSameIntermediateStreamTwice() { StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false); // should throw exception streamAppDesc.getIntermediateStream("test-stream-1", mock(Serde.class), false); }
@Test public void testGetOutputStreamWithKeyValueSerde() { String streamId = "test-stream-1"; KVSerde mockKVSerde = mock(KVSerde.class); Serde mockKeySerde = mock(Serde.class); Serde mockValueSerde = mock(Serde.class); doReturn(mockKeySerde).when(mockKVSerde).getKeySerde(); doReturn(mockValueSerde).when(mockKVSerde).getValueSerde(); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockKVSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd); }, getConfig()); OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId); assertEquals(streamId, outputStreamImpl.getStreamId()); assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId)); assertEquals(mockKeySerde, outputStreamImpl.getKeySerde()); assertEquals(mockValueSerde, outputStreamImpl.getValueSerde()); }
@Test(expected = SamzaException.class) public void testGetNextOpIdRejectsDuplicates() { HashMap<String, String> configMap = new HashMap<>(); configMap.put(JobConfig.JOB_NAME(), "jobName"); configMap.put(JobConfig.JOB_ID(), "1234"); Config config = new MapConfig(configMap); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, config); assertEquals("jobName-1234-join-customName", streamAppDesc.getNextOpId(OpCode.JOIN, "customName")); streamAppDesc.getNextOpId(OpCode.JOIN, "customName"); // should throw }
@Test public void testConstructor() { StreamApplication mockApp = mock(StreamApplication.class); Config mockConfig = getConfig(); StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(mockApp, mockConfig); verify(mockApp).describe(appDesc); assertEquals(mockConfig, appDesc.getConfig()); }
this.outputStrmMap = new LinkedHashMap<>(); outputStrmMap.put(outputStreamId, outputStream1); when(mockAppDesc.getInputOperators()).thenReturn(Collections.unmodifiableMap(inputOpSpecMap)); when(mockAppDesc.getOutputStreams()).thenReturn(Collections.unmodifiableMap(outputStrmMap)); this.allOpSpecs = new HashSet<OperatorSpec>() { { this.add(testInput);
@Test public void testApplicationTaskContextFactory() { ApplicationTaskContextFactory factory = mock(ApplicationTaskContextFactory.class); StreamApplication testApp = appDesc -> appDesc.withApplicationTaskContextFactory(factory); StreamApplicationDescriptorImpl appSpec = new StreamApplicationDescriptorImpl(testApp, getConfig()); assertEquals(appSpec.getApplicationTaskContextFactory(), Optional.of(factory)); }