@Test public void testViewUnboundedAsListDirect() { testViewUnbounded(pipeline, View.asList()); }
@Test public void testViewNonmergingAsListDirect() { testViewNonmerging(pipeline, View.asList()); }
@Test public void testViewUnboundedAsListBatch() { testViewUnbounded(createTestBatchRunner(), View.asList()); }
@Test public void testViewNonmergingAsListStreaming() { testViewNonmerging(createTestStreamingRunner(), View.asList()); }
@Test public void testViewNonmergingAsListBatch() { testViewNonmerging(createTestBatchRunner(), View.asList()); }
@Test public void testViewUnboundedAsListStreaming() { testViewUnbounded(createTestStreamingRunner(), View.asList()); }
private <SignalT> PCollectionView<?> expandTyped(PCollection<SignalT> input) { return input .apply(Window.<SignalT>configure().triggering(Never.ever()).discardingFiredPanes()) // Perform a per-window pre-combine so that our performance does not critically depend // on combiner lifting. .apply(ParDo.of(new CollectWindowsFn<>())) .apply(Sample.any(1)) .apply(View.asList()); } }
.apply("asList", View.asList()); final PCollectionView<Map<String, String>> sideInput2 = p.apply("Create SideInput2", Create.of(KV.of("a", "a"), KV.of("b", "b"), KV.of("c", "c")))
@Test public void testListSideInputTranslation() throws Exception { assertEquals( ListCoder.of(KvCoder.of(VoidCoder.of(), VarIntCoder.of())), getTranslatedSideInputCoder(ImmutableList.of(11, 13, 17, 23), View.asList())); }
@Test public void replacementGetViewReturnsOriginal() { final PCollection<Integer> ints = p.apply("CreateContents", Create.of(1, 2, 3)); final PCollectionView<List<Integer>> view = ints.apply(View.asList()); PTransformReplacement<PCollection<Integer>, PCollection<Integer>> replacement = factory.getReplacementTransform(
.apply("Create side input", Create.of(42)) .apply("Window side input", Window.into(FixedWindows.of(Duration.millis(10)))) .apply("View side input", View.asList());
final PCollectionView<List<Long>> windowedView = windowedElements.apply(View.asList());
@Test public void getViewsReturnsViews() { PCollectionView<List<String>> listView = p.apply("listCreate", Create.of("foo", "bar")) .apply( ParDo.of( new DoFn<String, String>() { @ProcessElement public void processElement(DoFn<String, String>.ProcessContext c) throws Exception { c.output(Integer.toString(c.element().length())); } })) .apply(View.asList()); PCollectionView<Object> singletonView = p.apply("singletonCreate", Create.<Object>of(1, 2, 3)).apply(View.asSingleton()); p.replaceAll( DirectRunner.fromOptions(TestPipeline.testingPipelineOptions()) .defaultTransformOverrides()); p.traverseTopologically(visitor); assertThat(visitor.getGraph().getViews(), Matchers.containsInAnyOrder(listView, singletonView)); }
@Test @Category(ValidatesRunner.class) public void testListSideInput() { final PCollectionView<List<Integer>> view = pipeline.apply("CreateSideInput", Create.of(11, 13, 17, 23)).apply(View.asList()); PCollection<Integer> output = pipeline .apply("CreateMainInput", Create.of(29, 31)) .apply( "OutputSideInputs", ParDo.of( new DoFn<Integer, Integer>() { @ProcessElement public void processElement(ProcessContext c) { checkArgument(c.sideInput(view).size() == 4); checkArgument( c.sideInput(view).get(0).equals(c.sideInput(view).get(0))); for (Integer i : c.sideInput(view)) { c.output(i); } } }) .withSideInputs(view)); PAssert.that(output).containsInAnyOrder(11, 13, 17, 23, 11, 13, 17, 23); pipeline.run(); }
@Test @Category(ValidatesRunner.class) public void testEmptyListSideInput() throws Exception { final PCollectionView<List<Integer>> view = pipeline.apply("CreateEmptyView", Create.empty(VarIntCoder.of())).apply(View.asList()); PCollection<Integer> results = pipeline .apply("Create1", Create.of(1)) .apply( "OutputSideInputs", ParDo.of( new DoFn<Integer, Integer>() { @ProcessElement public void processElement(ProcessContext c) { assertTrue(c.sideInput(view).isEmpty()); assertFalse(c.sideInput(view).iterator().hasNext()); c.output(1); } }) .withSideInputs(view)); // Pass at least one value through to guarantee that DoFn executes. PAssert.that(results).containsInAnyOrder(1); pipeline.run(); }
@Test public void getReplacementTransformGetSideInputs() { PCollectionView<Long> sideLong = pipeline .apply("LongSideInputVals", Create.of(-1L, -2L, -4L)) .apply("SideLongView", Sum.longsGlobally().asSingletonView()); PCollectionView<List<String>> sideStrings = pipeline .apply("StringSideInputVals", Create.of("foo", "bar", "baz")) .apply("SideStringsView", View.asList()); ParDo.SingleOutput<Integer, Long> originalTransform = ParDo.of(new ToLongFn()).withSideInputs(sideLong, sideStrings); PCollection<? extends Integer> input = pipeline.apply(Create.of(1, 2, 3)); AppliedPTransform< PCollection<? extends Integer>, PCollection<Long>, ParDo.SingleOutput<Integer, Long>> application = AppliedPTransform.of( "original", input.expand(), input.apply(originalTransform).expand(), originalTransform, pipeline); PTransformReplacement<PCollection<? extends Integer>, PCollection<Long>> replacementTransform = factory.getReplacementTransform(application); ParDoSingle<Integer, Long> parDoSingle = (ParDoSingle<Integer, Long>) replacementTransform.getTransform(); assertThat(parDoSingle.getSideInputs(), containsInAnyOrder(sideStrings, sideLong)); }
pipeline.apply("CreateSideInput", Create.of(11)).apply(View.asList());
pipeline.apply("Create list for side input", Create.of(2, 1, 0)).apply(View.asList());
TimestampedValue.of(43, new Instant(11)))) .apply("SideWindowInto", Window.into(FixedWindows.of(Duration.millis(10)))) .apply(View.asList());