/** * Returns a {@link TupleTagList} containing the given {@link TupleTag TupleTags}, in order. * * <p>Longer {@link TupleTagList TupleTagLists} can be created by calling {@link #and} on the * result. */ public static TupleTagList of(List<TupleTag<?>> tags) { return empty().and(tags); }
public TupleTagList getTags() { TupleTagList list = TupleTagList.empty(); for (TupleTag<FeatureRowExtended> tag : tagMap.values()) { list = list.and(tag); } return list; }
KeyedPCollectionTuple( Pipeline pipeline, List<TaggedKeyedPCollection<K, ?>> keyedCollections, TupleTagList tupleTagList, @Nullable Coder<K> keyCoder) { this.pipeline = pipeline; this.keyedCollections = keyedCollections; this.schema = new CoGbkResultSchema(tupleTagList); this.keyCoder = keyCoder; }
/** * Returns a singleton {@link TupleTagList} containing the given {@link TupleTag}. * * <p>Longer {@link TupleTagList TupleTagLists} can be created by calling {@link #and} on the * result. */ public static TupleTagList of(TupleTag<?> tag) { return empty().and(tag); }
public static CoGbkResultSchema of(List<TupleTag<?>> tags) { TupleTagList tupleTags = TupleTagList.empty(); for (TupleTag<?> tag : tags) { tupleTags = tupleTags.and(tag); } return new CoGbkResultSchema(tupleTags); }
/** Returns an empty {@link CoGbkResult}. */ public static <V> CoGbkResult empty() { return new CoGbkResult( new CoGbkResultSchema(TupleTagList.empty()), new ArrayList<Iterable<?>>()); }
@Override public PCollection<OutputT> expand(PCollection<? extends InputT> input) { TupleTag<OutputT> mainOutput = new TupleTag<>(); return input.apply(original.withOutputTags(mainOutput, TupleTagList.empty())).get(mainOutput); } }
@Override public PCollectionTuple expand(PCollection<PublishRequest> input) { checkArgument(getTopicName() != null, "withTopicName() is required"); return input.apply( ParDo.of(new SnsWriterFn(this)) .withOutputTags(getResultOutputTag(), TupleTagList.empty())); }
@Before public void setup() { MockitoAnnotations.initMocks(this); inputPc = p.apply(Create.of(1, 2, 3)); mainOutputTag = new TupleTag<Integer>() {}; additionalOutputTags = TupleTagList.empty().getAll(); bundleFactory = ImmutableListBundleFactory.create(); }
private ParDo.MultiOutput<PubsubMessage, Row> createParserParDo() { return ParDo.of( PubsubMessageToRow.builder() .messageSchema(getSchema()) .useDlq(getDeadLetterQueue() != null) .build()) .withOutputTags(MAIN_TAG, useDlq() ? TupleTagList.of(DLQ_TAG) : TupleTagList.empty()); }
@Test public void parDoWithState() { AppliedPTransform<?, ?, ?> statefulApplication = getAppliedTransform( ParDo.of(doFnWithState).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDo().matches(statefulApplication), is(true)); AppliedPTransform<?, ?, ?> splittableApplication = getAppliedTransform( ParDo.of(splittableDoFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDo().matches(splittableApplication), is(false)); }
@Test public void parDoMulti() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform(ParDo.of(doFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void parDoMultiWithState() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(doFnWithState).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void parDoMultiWithTimers() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(doFnWithTimers).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void parDoSplittable() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(splittableDoFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.splittableParDo().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void parDoMultiSplittable() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(splittableDoFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void retainOnlyPrimitivesWithOnlyPrimitivesUnchanged() { Pipeline p = Pipeline.create(); p.apply("Read", Read.from(CountingSource.unbounded())) .apply( "multi-do", ParDo.of(new TestFn()).withOutputTags(new TupleTag<>(), TupleTagList.empty())); Components originalComponents = PipelineTranslation.toProto(p).getComponents(); Collection<String> primitiveComponents = QueryablePipeline.getPrimitiveTransformIds(originalComponents); assertThat(primitiveComponents, equalTo(originalComponents.getTransformsMap().keySet())); }
private PCollection<String> applySplittableParDo( String name, PCollection<Integer> input, DoFn<Integer, String> fn) { ParDo.MultiOutput<Integer, String> multiOutput = ParDo.of(fn).withOutputTags(MAIN_OUTPUT_TAG, TupleTagList.empty()); PCollectionTuple output = multiOutput.expand(input); output.get(MAIN_OUTPUT_TAG).setName("main"); AppliedPTransform<PCollection<Integer>, PCollectionTuple, ?> transform = AppliedPTransform.of("ParDo", input.expand(), output.expand(), multiOutput, pipeline); return input.apply(name, SplittableParDo.forAppliedParDo(transform)).get(MAIN_OUTPUT_TAG); }
@Test @Category({ValidatesRunner.class, UsesParDoLifecycle.class}) public void testFnCallSequenceMulti() { PCollectionList.of(p.apply("Impolite", Create.of(1, 2, 4))) .and(p.apply("Polite", Create.of(3, 5, 6, 7))) .apply(Flatten.pCollections()) .apply( ParDo.of(new CallSequenceEnforcingFn<Integer>()) .withOutputTags(new TupleTag<Integer>() {}, TupleTagList.empty())); p.run(); }
@Test @Category({ValidatesRunner.class, UsesStatefulParDo.class, UsesParDoLifecycle.class}) public void testFnCallSequenceStateful() { PCollectionList.of(p.apply("Impolite", Create.of(KV.of("a", 1), KV.of("b", 2), KV.of("a", 4)))) .and( p.apply( "Polite", Create.of(KV.of("b", 3), KV.of("a", 5), KV.of("c", 6), KV.of("c", 7)))) .apply(Flatten.pCollections()) .apply( ParDo.of(new CallSequenceEnforcingStatefulFn<String, Integer>()) .withOutputTags(new TupleTag<KV<String, Integer>>() {}, TupleTagList.empty())); p.run(); }