/** * Call this constructor to initialize the fields for which this base class provides boilerplate * accessors, with an auto-generated tag. */ private SimplePCollectionView( PCollection<ElemT> pCollection, ViewFn<PrimitiveViewT, ViewT> viewFn, WindowMappingFn<W> windowMappingFn, WindowingStrategy<?, W> windowingStrategy) { this(pCollection, new TupleTag<>(), viewFn, windowMappingFn, windowingStrategy); }
private CoGbkResultSchema createSchema(int size) { List<TupleTag<?>> tags = new ArrayList<>(); for (int i = 0; i < size; i++) { tags.add(new TupleTag<Integer>("tag" + i)); } return new CoGbkResultSchema(TupleTagList.of(tags)); }
private WriteResult writeResult(Pipeline p) { PCollection<TableRow> empty = p.apply("CreateEmptyFailedInserts", Create.empty(TypeDescriptor.of(TableRow.class))); return WriteResult.in(p, new TupleTag<>("failedInserts"), empty); }
@Test public void testUntypedMainOutputTagTypedOutputTupleTag() { // mainOutputTag is allowed to be untyped because Coder can be inferred other ways. TupleTag<Integer> mainOutputTag = new TupleTag<>(); TupleTag<Integer> typedOutputTag = new TupleTag<Integer>() {}; PCollectionTuple tuple = buildPCollectionTupleWithTags(mainOutputTag, typedOutputTag); assertThat(tuple.get(typedOutputTag).getCoder(), instanceOf(VarIntCoder.class)); }
public void runLazyResult(int cacheSize) { int valueLen = 7; TestUnionValues values = new TestUnionValues(0, 1, 0, 3, 0, 3, 3); CoGbkResult result = new CoGbkResult(createSchema(5), values, cacheSize); assertThat(values.maxPos(), equalTo(Math.min(cacheSize, valueLen))); assertThat(result.getAll(new TupleTag<>("tag0")), contains(0, 2, 4)); assertThat(values.maxPos(), equalTo(valueLen)); assertThat(result.getAll(new TupleTag<>("tag3")), contains(3, 5, 6)); assertThat(result.getAll(new TupleTag<Integer>("tag2")), emptyIterable()); assertThat(result.getOnly(new TupleTag<>("tag1")), equalTo(1)); assertThat(result.getAll(new TupleTag<>("tag0")), contains(0, 2, 4)); }
@Test public void testUntypedOutputTupleTagGivesActionableMessage() { TupleTag<Integer> mainOutputTag = new TupleTag<Integer>() {}; // untypedOutputTag did not use anonymous subclass. TupleTag<Integer> untypedOutputTag = new TupleTag<>(); PCollectionTuple tuple = buildPCollectionTupleWithTags(mainOutputTag, untypedOutputTag); thrown.expect(IllegalStateException.class); thrown.expectMessage("No Coder has been manually specified"); thrown.expectMessage("Building a Coder using a registered CoderProvider failed"); Coder<?> coder = tuple.get(untypedOutputTag).getCoder(); System.out.println(coder); }
@Test public void testEmpty() { TupleTag<Object> tag = new TupleTag<>(); assertFalse(PCollectionTuple.empty(pipeline).has(tag)); }
private void runCompositeFilter(PCollection<Row> input) throws Exception { String sql = "SELECT * FROM TABLE_A" + " WHERE f_int > 1 AND (f_long < 3000 OR f_string = 'string_row3')"; PCollection<Row> result = PCollectionTuple.of(new TupleTag<>("TABLE_A"), input) .apply("testCompositeFilter", SqlTransform.query(sql)); PAssert.that(result).containsInAnyOrder(rowsInTableA.get(1), rowsInTableA.get(2)); pipeline.run().waitUntilFinish(); }
@Test public void parDoWithState() { AppliedPTransform<?, ?, ?> statefulApplication = getAppliedTransform( ParDo.of(doFnWithState).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDo().matches(statefulApplication), is(true)); AppliedPTransform<?, ?, ?> splittableApplication = getAppliedTransform( ParDo.of(splittableDoFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDo().matches(splittableApplication), is(false)); }
@Test public void parDoMulti() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform(ParDo.of(doFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void parDoMultiWithState() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(doFnWithState).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void parDoMultiWithTimers() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(doFnWithTimers).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.splittableParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
@Test public void testProjectUnknownField() throws Exception { exceptions.expect(ParseException.class); exceptions.expectCause(hasMessage(containsString("Column 'f_int_na' not found in any table"))); pipeline.enableAbandonedNodeEnforcement(false); String sql = "SELECT f_int_na FROM TABLE_A"; PCollection<Row> result = PCollectionTuple.of(new TupleTag<>("TABLE_A"), boundedInput1) .apply("testProjectUnknownField", SqlTransform.query(sql)); pipeline.run().waitUntilFinish(); } }
@Test public void testCoderIsSerializableWithWellKnownCoderType() { CoderProperties.coderSerializable( CoGbkResultCoder.of( CoGbkResultSchema.of(ImmutableList.of(new TupleTag<GlobalWindow>())), UnionCoder.of(ImmutableList.of(GlobalWindow.Coder.INSTANCE)))); } }
@Test public void retainOnlyPrimitivesWithOnlyPrimitivesUnchanged() { Pipeline p = Pipeline.create(); p.apply("Read", Read.from(CountingSource.unbounded())) .apply( "multi-do", ParDo.of(new TestFn()).withOutputTags(new TupleTag<>(), TupleTagList.empty())); Components originalComponents = PipelineTranslation.toProto(p).getComponents(); Collection<String> primitiveComponents = QueryablePipeline.getPrimitiveTransformIds(originalComponents); assertThat(primitiveComponents, equalTo(originalComponents.getTransformsMap().keySet())); }
@Test public void testDuplicatedTags() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("it is already present in the composition"); TupleTag<Integer> tag = new TupleTag<>(); CombineFns.compose() .with(new GetIntegerFunction(), Max.ofIntegers(), tag) .with(new GetIntegerFunction(), Min.ofIntegers(), tag); }
/** Tests that Pipeline supports pulling an element out of a tuple as a transform. */ @Test @Category(ValidatesRunner.class) public void testTupleProjectionTransform() throws Exception { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3, 4)); TupleTag<Integer> tag = new TupleTag<>(); PCollectionTuple tuple = PCollectionTuple.of(tag, input); PCollection<Integer> output = tuple.apply("ProjectTag", new TupleProjectionTransform<>(tag)); PAssert.that(output).containsInAnyOrder(1, 2, 3, 4); pipeline.run(); }
/** Tests that Pipeline supports putting an element into a tuple as a transform. */ @Test @Category(ValidatesRunner.class) public void testTupleInjectionTransform() throws Exception { PCollection<Integer> input = pipeline.apply(Create.of(1, 2, 3, 4)); TupleTag<Integer> tag = new TupleTag<>(); PCollectionTuple output = input.apply("ProjectTag", new TupleInjectionTransform<>(tag)); PAssert.that(output.get(tag)).containsInAnyOrder(1, 2, 3, 4); pipeline.run(); }
@Test public void testDuplicatedTagsWithContext() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("it is already present in the composition"); TupleTag<UserString> tag = new TupleTag<>(); CombineFns.compose() .with(new GetUserStringFunction(), new ConcatStringWithContext(null /* view */), tag) .with(new GetUserStringFunction(), new ConcatStringWithContext(null /* view */), tag); }
public void runLiteralField(PCollection<Row> input) throws Exception { String sql = "SELECT 1 as literal_field FROM TABLE_A"; PCollection<Row> result = PCollectionTuple.of(new TupleTag<>("TABLE_A"), input) .apply("testLiteralField", SqlTransform.query(sql)); Schema resultType = Schema.builder().addInt32Field("literal_field").build(); Row row = Row.withSchema(resultType).addValues(1).build(); PAssert.that(result).containsInAnyOrder(row); pipeline.run().waitUntilFinish(); }