Collection<Integer> inputIds = transform(transform.getInput());
@Test public void readSourceTranslatorUnboundedWithoutMaxParallelism() { final int parallelism = 2; Read.Unbounded transform = Read.from(new TestUnboundedSource()); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); StreamTransformation<?> sourceTransform = applyReadSourceTransform(transform, PCollection.IsBounded.UNBOUNDED, env); UnboundedSourceWrapper source = (UnboundedSourceWrapper) ((SourceTransformation) ((OneInputTransformation) sourceTransform).getInput()) .getOperator() .getUserFunction(); assertEquals(parallelism, source.getSplitSources().size()); }
/** * Transforms a {@code OneInputTransformation}. * * <p>This recursively transforms the inputs, creates a new {@code StreamNode} in the graph and * wired the inputs to this new node. */ private <IN, OUT> Collection<Integer> transformOneInputTransform(OneInputTransformation<IN, OUT> transform) { Collection<Integer> inputIds = transform(transform.getInput()); // the recursive call might have already transformed this if (alreadyTransformed.containsKey(transform)) { return alreadyTransformed.get(transform); } String slotSharingGroup = determineSlotSharingGroup(transform.getSlotSharingGroup(), inputIds); streamGraph.addOperator(transform.getId(), slotSharingGroup, transform.getOperator(), transform.getInputType(), transform.getOutputType(), transform.getName()); if (transform.getStateKeySelector() != null) { TypeSerializer<?> keySerializer = transform.getStateKeyType().createSerializer(env.getConfig()); streamGraph.setOneInputStateKey(transform.getId(), transform.getStateKeySelector(), keySerializer); } streamGraph.setParallelism(transform.getId(), transform.getParallelism()); streamGraph.setMaxParallelism(transform.getId(), transform.getMaxParallelism()); for (Integer inputId: inputIds) { streamGraph.addEdge(inputId, transform.getId(), 0); } return Collections.singleton(transform.getId()); }
@Test public void readSourceTranslatorUnboundedWithMaxParallelism() { final int maxParallelism = 6; final int parallelism = 2; Read.Unbounded transform = Read.from(new TestUnboundedSource()); StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(parallelism); env.setMaxParallelism(maxParallelism); StreamTransformation<?> sourceTransform = applyReadSourceTransform(transform, PCollection.IsBounded.UNBOUNDED, env); UnboundedSourceWrapper source = (UnboundedSourceWrapper) ((SourceTransformation) ((OneInputTransformation) sourceTransform).getInput()) .getOperator() .getUserFunction(); assertEquals(maxParallelism, source.getSplitSources().size()); }
Collection<Integer> inputIds = transform(transform.getInput());
Collection<Integer> inputIds = transform(transform.getInput());