public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
@Override public void validateStream(StreamSpec streamSpec) throws StreamValidationException { LOG.info("About to validate stream = " + streamSpec); String streamName = streamSpec.getPhysicalName(); SystemStreamMetadata systemStreamMetadata = getSystemStreamMetadata(Collections.singleton(streamName)).get(streamName); if (systemStreamMetadata == null) { throw new StreamValidationException( "Failed to obtain metadata for stream " + streamName + ". Validation failed."); } int actualPartitionCounter = systemStreamMetadata.getSystemStreamPartitionMetadata().size(); int expectedPartitionCounter = streamSpec.getPartitionCount(); LOG.info("actualCount=" + actualPartitionCounter + "; expectedCount=" + expectedPartitionCounter); if (actualPartitionCounter != expectedPartitionCounter) { throw new StreamValidationException( String.format("Mismatch of partitions for stream %s. Expected %d, got %d. Validation failed.", streamName, expectedPartitionCounter, actualPartitionCounter)); } }
/** * Populate the metadata for the {@link SystemStream} and initialize the buffer for {@link SystemStreamPartition}. * * @param streamSpec stream spec for the stream to be initialized * * @return true if successful, false otherwise */ boolean initializeStream(StreamSpec streamSpec) { LOG.info("Initializing the stream for {}", streamSpec.getId()); systemStreamToPartitions.put(streamSpec.toSystemStream(), streamSpec.getPartitionCount()); for (int partition = 0; partition < streamSpec.getPartitionCount(); partition++) { bufferedMessages.put( new SystemStreamPartition(streamSpec.toSystemStream(), new Partition(partition)), newSynchronizedLinkedList()); } return true; }
/** * Populate the metadata for the {@link SystemStream} and initialize the buffer for {@link SystemStreamPartition}. * * @param streamSpec stream spec for the stream to be initialized * * @return true if successful, false otherwise */ boolean initializeStream(StreamSpec streamSpec) { LOG.info("Initializing the stream for {}", streamSpec.getId()); systemStreamToPartitions.put(streamSpec.toSystemStream(), streamSpec.getPartitionCount()); for (int partition = 0; partition < streamSpec.getPartitionCount(); partition++) { bufferedMessages.put( new SystemStreamPartition(streamSpec.toSystemStream(), new Partition(partition)), newSynchronizedLinkedList()); } return true; }
/** * Populate the metadata for the {@link SystemStream} and initialize the buffer for {@link SystemStreamPartition}. * * @param streamSpec stream spec for the stream to be initialized * * @return true if successful, false otherwise */ boolean initializeStream(StreamSpec streamSpec) { LOG.info("Initializing the stream for {}", streamSpec.getId()); systemStreamToPartitions.put(streamSpec.toSystemStream(), streamSpec.getPartitionCount()); for (int partition = 0; partition < streamSpec.getPartitionCount(); partition++) { bufferedMessages.put( new SystemStreamPartition(streamSpec.toSystemStream(), new Partition(partition)), newSynchronizedLinkedList()); } return true; }
/** * Populate the metadata for the {@link SystemStream} and initialize the buffer for {@link SystemStreamPartition}. * * @param streamSpec stream spec for the stream to be initialized * * @return true if successful, false otherwise */ boolean initializeStream(StreamSpec streamSpec) { LOG.info("Initializing the stream for {}", streamSpec.getId()); systemStreamToPartitions.put(streamSpec.toSystemStream(), streamSpec.getPartitionCount()); for (int partition = 0; partition < streamSpec.getPartitionCount(); partition++) { bufferedMessages.put( new SystemStreamPartition(streamSpec.toSystemStream(), new Partition(partition)), newSynchronizedLinkedList()); } return true; }
/** * Populate the metadata for the {@link SystemStream} and initialize the buffer for {@link SystemStreamPartition}. * * @param streamSpec stream spec for the stream to be initialized * * @return true if successful, false otherwise */ boolean initializeStream(StreamSpec streamSpec) { LOG.info("Initializing the stream for {}", streamSpec.getId()); systemStreamToPartitions.put(streamSpec.toSystemStream(), streamSpec.getPartitionCount()); for (int partition = 0; partition < streamSpec.getPartitionCount(); partition++) { bufferedMessages.put( new SystemStreamPartition(streamSpec.toSystemStream(), new Partition(partition)), newSynchronizedLinkedList()); } return true; }
@Test public void testDefaultPartitions() { Map<String, String> map = new HashMap<>(config); map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS)); Config cfg = new MapConfig(map); ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager); StreamApplicationDescriptorImpl graphSpec = createSimpleGraph(); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input1 jobGraph.getIntermediateStreams().forEach(edge -> { assertTrue(edge.getPartitionCount() == DEFAULT_PARTITIONS); }); }
@Test public void testCalculateJoinInputPartitions() { ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamStreamJoin(); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input1 jobGraph.getIntermediateStreams().forEach(edge -> { assertEquals(64, edge.getPartitionCount()); }); }
@Test public void testCalculateInStreamPartitionsBehindTablesWithSideInputs() { ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamTableJoinWithSideInputs(); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input1 jobGraph.getIntermediateStreams().forEach(edge -> { assertEquals(64, edge.getPartitionCount()); }); }
@Test public void testCalculateIntStreamPartitions() { ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); StreamApplicationDescriptorImpl graphSpec = createSimpleGraph(); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input1 jobGraph.getIntermediateStreams().forEach(edge -> { assertEquals(64, edge.getPartitionCount()); // max of input1 and output1 }); }
@Test public void testCalculateInStreamPartitionsBehindTables() { ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithStreamTableJoin(); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input3 jobGraph.getIntermediateStreams().forEach(edge -> { assertEquals(32, edge.getPartitionCount()); }); }
@Test public void testCalculateOrderSensitiveJoinInputPartitions() { // This test ensures that the ExecutionPlanner can handle groups of joined stream edges // in the correct order. It creates an example stream-stream join application that has // the following sets of joined streams (notice the order): // // a. e1 (16), e2` (?) // b. e3` (?), e4` (?) // c. e2` (?), e3` (?) // // If processed in the above order, the ExecutionPlanner will fail to assign the partitions // correctly. ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); StreamApplicationDescriptorImpl graphSpec = createStreamGraphWithComplexStreamStreamJoin(); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input1 jobGraph.getIntermediateStreams().forEach(edge -> { assertEquals(64, edge.getPartitionCount()); }); }
@Test public void testGetStreamSpec() { StreamEdge edge = new StreamEdge(spec, false, false, new MapConfig()); assertEquals(edge.getStreamSpec(), spec); assertEquals(edge.getStreamSpec().getPartitionCount(), 1 /*StreamSpec.DEFAULT_PARTITION_COUNT*/); edge.setPartitionCount(10); assertEquals(edge.getStreamSpec().getPartitionCount(), 10); }
@Test public void testMaxPartitionLimit() { int partitionLimit = IntermediateStreamManager.MAX_INFERRED_PARTITIONS; ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); StreamApplicationDescriptorImpl graphSpec = new StreamApplicationDescriptorImpl(appDesc -> { MessageStream<KV<Object, Object>> input1 = appDesc.getInputStream(input4Descriptor); OutputStream<KV<Object, Object>> output1 = appDesc.getOutputStream(output1Descriptor); input1.partitionBy(m -> m.key, m -> m.value, mock(KVSerde.class), "p1").map(kv -> kv).sendTo(output1); }, config); JobGraph jobGraph = (JobGraph) planner.plan(graphSpec); // Partitions should be the same as input1 jobGraph.getIntermediateStreams().forEach(edge -> { assertEquals(partitionLimit, edge.getPartitionCount()); // max of input1 and output1 }); }
@Test public void testBasicConstructor() { StreamSpec streamSpec = new StreamSpec("dummyId", "dummyPhysicalName", "dummySystemName", 1); assertEquals("dummyId", streamSpec.getId()); assertEquals("dummyPhysicalName", streamSpec.getPhysicalName()); assertEquals("dummySystemName", streamSpec.getSystemName()); assertEquals(1, streamSpec.getPartitionCount()); // SystemStream should use the physical name, not the streamId. SystemStream systemStream = new SystemStream("dummySystemName", "dummyPhysicalName"); assertEquals(systemStream, streamSpec.toSystemStream()); }