@Test public void pushWithoutPushFails() { thrown.expect(IllegalStateException.class); hierarchy.popNode(); }
/** * Applies a {@link PTransform} to the given {@link PInput}. * * @see Pipeline#apply */ private <InputT extends PInput, OutputT extends POutput> OutputT applyInternal( String name, InputT input, PTransform<? super InputT, OutputT> transform) { String namePrefix = transforms.getCurrent().getFullName(); String uniqueName = uniquifyInternal(namePrefix, name); final String builtName = buildName(namePrefix, name); instancePerName.put(builtName, transform); LOG.debug("Adding {} to {}", transform, this); transforms.pushNode(uniqueName, input, transform); try { transforms.finishSpecifyingInput(); OutputT output = transform.expand(input); transforms.setOutput(output); return output; } finally { transforms.popNode(); } }
@Test public void pushThenPopSucceeds() { TransformHierarchy.Node root = hierarchy.getCurrent(); TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); assertThat(hierarchy.getCurrent(), equalTo(node)); hierarchy.popNode(); assertThat(node.finishedSpecifying, is(true)); assertThat(hierarchy.getCurrent(), equalTo(root)); }
private < InputT extends PInput, OutputT extends POutput, TransformT extends PTransform<? super InputT, OutputT>> void applyReplacement( Node original, PTransformOverrideFactory<InputT, OutputT, TransformT> replacementFactory) { PTransformReplacement<InputT, OutputT> replacement = replacementFactory.getReplacementTransform( (AppliedPTransform<InputT, OutputT, TransformT>) original.toAppliedPTransform(this)); if (replacement.getTransform() == original.getTransform()) { return; } InputT originalInput = replacement.getInput(); LOG.debug("Replacing {} with {}", original, replacement); transforms.replaceNode(original, originalInput, replacement.getTransform()); try { OutputT newOutput = replacement.getTransform().expand(originalInput); Map<PValue, ReplacementOutput> originalToReplacement = replacementFactory.mapOutputs(original.getOutputs(), newOutput); // Ensure the internal TransformHierarchy data structures are consistent. transforms.setOutput(newOutput); transforms.replaceOutputs(originalToReplacement); } finally { transforms.popNode(); } }
hierarchy.pushNode("consumes_both", one, multiConsumer); hierarchy.setOutput(done); hierarchy.popNode(); hierarchy.pushNode("creates_one_and_two", PBegin.in(pipeline), producer); hierarchy.setOutput(oneAndTwo); hierarchy.popNode(); hierarchy.setOutput(oneAndTwo); hierarchy.popNode(); hierarchy.popNode();
@Test public void emptyCompositeSucceeds() { PCollection<Long> created = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); hierarchy.setOutput(created); hierarchy.popNode(); PCollectionList<Long> pcList = PCollectionList.of(created); TransformHierarchy.Node emptyTransform = hierarchy.pushNode( "Extract", pcList, new PTransform<PCollectionList<Long>, PCollection<Long>>() { @Override public PCollection<Long> expand(PCollectionList<Long> input) { return input.get(0); } }); hierarchy.setOutput(created); hierarchy.popNode(); assertThat(hierarchy.getProducer(created), equalTo(node)); assertThat( "A Transform that produces non-primitive output should be composite", emptyTransform.isCompositeNode(), is(true)); }
hierarchy.finishSpecifyingInput(); hierarchy.setOutput(upstream); hierarchy.popNode(); hierarchy.popNode(); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(replacementOutput); hierarchy.popNode(); hierarchy.setOutput(replacementOutput.get(longs)); equalTo(replacementOutput.get(longs).expand().keySet())); assertThat(compositeNode.getOutputs().values(), Matchers.contains(output)); hierarchy.popNode();
hierarchy.finishSpecifyingInput(); hierarchy.setOutput(upstream); hierarchy.popNode(); hierarchy.popNode(); hierarchy.finishSpecifyingInput(); hierarchy.setOutput(replacementOutput); hierarchy.popNode(); hierarchy.setOutput(replacementOutput.get(longs)); TaggedPValue.ofExpandedValue(output), TaggedPValue.of(replacementLongs.getKey(), replacementLongs.getValue())))); hierarchy.popNode();
@Test public void producingOwnOutputWithCompositeFails() { final PCollection<Long> comp = PCollection.createPrimitiveOutputInternal( pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of()); PTransform<PBegin, PCollection<Long>> root = new PTransform<PBegin, PCollection<Long>>() { @Override public PCollection<Long> expand(PBegin input) { return comp; } }; hierarchy.pushNode("Composite", PBegin.in(pipeline), root); Create.Values<Integer> create = Create.of(1); hierarchy.pushNode("Create", PBegin.in(pipeline), create); hierarchy.setOutput(pipeline.apply(create)); hierarchy.popNode(); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("contains a primitive POutput produced by it"); thrown.expectMessage("primitive transforms are permitted to produce"); thrown.expectMessage("Composite"); hierarchy.setOutput(comp); }
hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1)); hierarchy.setOutput(created); hierarchy.popNode(); PCollectionList<Long> pcList = PCollectionList.of(created);