/** * Applies a {@link PTransform} to the given {@link PInput}. * * @see Pipeline#apply */ private <InputT extends PInput, OutputT extends POutput> OutputT applyInternal( String name, InputT input, PTransform<? super InputT, OutputT> transform) { String namePrefix = transforms.getCurrent().getFullName(); String uniqueName = uniquifyInternal(namePrefix, name); final String builtName = buildName(namePrefix, name); instancePerName.put(builtName, transform); LOG.debug("Adding {} to {}", transform, this); transforms.pushNode(uniqueName, input, transform); try { transforms.finishSpecifyingInput(); OutputT output = transform.expand(input); transforms.setOutput(output); return output; } finally { transforms.popNode(); } }
@Test public void pushThenPopSucceeds() { TransformHierarchy.Node root = hierarchy.getCurrent(); TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); assertThat(hierarchy.getCurrent(), equalTo(node)); hierarchy.popNode(); assertThat(node.finishedSpecifying, is(true)); assertThat(hierarchy.getCurrent(), equalTo(root)); }
hierarchy.pushNode("consumes_both", one, multiConsumer); hierarchy.setOutput(done); hierarchy.popNode(); hierarchy.pushNode( "encloses_producer", PBegin.in(pipeline), hierarchy.pushNode("creates_one_and_two", PBegin.in(pipeline), producer); hierarchy.setOutput(oneAndTwo); hierarchy.popNode(); hierarchy.popNode(); hierarchy.pushNode("second_copy_of_consumes_both", one, multiConsumer); hierarchy.setOutput(done); hierarchy.popNode();
@Test public void emptyCompositeSucceeds() { PCollection<Long> created = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); hierarchy.setOutput(created); hierarchy.popNode(); PCollectionList<Long> pcList = PCollectionList.of(created); TransformHierarchy.Node emptyTransform = hierarchy.pushNode( "Extract", pcList, new PTransform<PCollectionList<Long>, PCollection<Long>>() { @Override public PCollection<Long> expand(PCollectionList<Long> input) { return input.get(0); } }); hierarchy.setOutput(created); hierarchy.popNode(); assertThat(hierarchy.getProducer(created), equalTo(node)); assertThat( "A Transform that produces non-primitive output should be composite", emptyTransform.isCompositeNode(), is(true)); }
final PCollectionTuple oneAndTwo = PCollectionTuple.of(oneTag, one).and(twoTag, two); hierarchy.pushNode( "consumes_both", one, hierarchy.pushNode( "encloses_producer", PBegin.in(pipeline), Node enclosed = hierarchy.pushNode("creates_one_and_two", PBegin.in(pipeline), producer); hierarchy.setOutput(oneAndTwo); hierarchy.popNode();
pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); TransformHierarchy.Node compositeNode = hierarchy.pushNode("Create", begin, create); hierarchy.finishSpecifyingInput(); assertThat(hierarchy.getCurrent(), equalTo(compositeNode)); assertThat(compositeNode.getEnclosingNode().isRootNode(), is(true)); TransformHierarchy.Node primitiveNode = hierarchy.pushNode("Create/Read", begin, read); assertThat(hierarchy.getCurrent(), equalTo(primitiveNode)); hierarchy.finishSpecifyingInput(); hierarchy.popNode(); TransformHierarchy.Node otherPrimitive = hierarchy.pushNode("ParDo", created, pardo); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(mapped);
@Test public void producingOwnOutputWithCompositeFails() { final PCollection<Long> comp = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); PTransform<PBegin, PCollection<Long>> root = new PTransform<PBegin, PCollection<Long>>() { @Override public PCollection<Long> expand(PBegin input) { return comp; } }; hierarchy.pushNode("Composite", PBegin.in(pipeline), root); Create.Values<Integer> create = Create.of(1); hierarchy.pushNode("Create", PBegin.in(pipeline), create); hierarchy.setOutput(pipeline.apply(create)); hierarchy.popNode(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("contains a primitive POutput produced by it"); thrown.expectMessage("primitive transforms are permitted to produce"); thrown.expectMessage("Composite"); hierarchy.setOutput(comp); }
PCollection<Long> upstream = pipeline.apply(genUpstream); PCollection<Long> output = upstream.apply("Original", originalParDo); hierarchy.pushNode("Upstream", pipeline.begin(), genUpstream); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(upstream); hierarchy.popNode(); TransformHierarchy.Node original = hierarchy.pushNode("Original", upstream, originalParDo); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(output); Node replacementParNode = hierarchy.pushNode("Original/Contained", upstream, replacementParDo); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(replacementOutput);
PCollection<Long> upstream = pipeline.apply(genUpstream); PCollection<Long> output = upstream.apply("Original", originalParDo); Node upstreamNode = hierarchy.pushNode("Upstream", pipeline.begin(), genUpstream); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(upstream); hierarchy.popNode(); Node original = hierarchy.pushNode("Original", upstream, originalParDo); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(output); Node replacementParNode = hierarchy.pushNode("Original/Contained", upstream, replacementParDo); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(replacementOutput);
PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); hierarchy.setOutput(created); hierarchy.popNode(); VarLongCoder.of()) .setName("prim")); hierarchy.pushNode( "AddPc", pcList,