/** Like {@link #requiresSideInputs(Collection)}. */ public static Requirements requiresSideInputs(PCollectionView<?>... sideInputs) { return requiresSideInputs(Arrays.asList(sideInputs)); }
public static Requirements union(Contextful... contextfuls) { Set<PCollectionView<?>> sideInputs = Sets.newHashSet(); for (Contextful c : contextfuls) { sideInputs.addAll(c.getRequirements().getSideInputs()); } return requiresSideInputs(sideInputs); } }
@Override public PCollection<T> expand(PCollection<T> input) { List<PCollectionView<?>> views = Lists.newArrayList(); for (int i = 0; i < signals.size(); ++i) { views.add(signals.get(i).apply("To wait view " + i, new ToWaitView())); } return input.apply( "Wait", MapElements.into(input.getCoder().getEncodedTypeDescriptor()) .via(fn((t, c) -> t, requiresSideInputs(views)))); } }
requiresSideInputs(schemaView))) .via( fn( return AvroIO.sinkViaGenericRecords(schema, formatter); }, requiresSideInputs(schemaView))) .to(baseDir.toString()) .withNaming( requiresSideInputs(schemaView))) .withTempDirectory(baseDir.toString()) .withDestinationCoder(StringUtf8Coder.of())
@Test @Category(NeedsRunner.class) public void testSinglePollMultipleInputsWithSideInput() { final PCollectionView<String> moo = p.apply("moo", Create.of("moo")).apply("moo singleton", View.asSingleton()); final PCollectionView<String> zoo = p.apply("zoo", Create.of("zoo")).apply("zoo singleton", View.asSingleton()); PCollection<KV<String, String>> res = p.apply("input", Create.of("a", "b")) .apply( Watch.growthOf( new PollFn<String, String>() { @Override public PollResult<String> apply(String element, Context c) throws Exception { return PollResult.complete( Instant.now(), Arrays.asList( element + " " + c.sideInput(moo) + " " + c.sideInput(zoo))); } }, requiresSideInputs(moo, zoo)) .withPollInterval(Duration.ZERO)); PAssert.that(res) .containsInAnyOrder(Arrays.asList(KV.of("a", "a moo zoo"), KV.of("b", "b moo zoo"))); p.run(); }
/** Basic test of {@link MapElements} with a {@link Fn} and a side input. */ @Test @Category(NeedsRunner.class) public void testMapBasicWithSideInput() throws Exception { final PCollectionView<Integer> view = pipeline.apply("Create base", Create.of(40)).apply(View.asSingleton()); PCollection<Integer> output = pipeline .apply(Create.of(0, 1, 2)) .apply( MapElements.into(integers()) .via( fn((element, c) -> element + c.sideInput(view), requiresSideInputs(view)))); PAssert.that(output).containsInAnyOrder(40, 41, 42); pipeline.run(); }
/** Basic test of {@link FlatMapElements} with a {@link Fn} and a side input. */ @Test @Category(NeedsRunner.class) public void testFlatMapBasicWithSideInput() throws Exception { final PCollectionView<Integer> view = pipeline.apply("Create base", Create.of(40)).apply(View.asSingleton()); PCollection<Integer> output = pipeline .apply(Create.of(0, 1, 2)) .apply( FlatMapElements.into(integers()) .via( fn( (input, c) -> ImmutableList.of( c.sideInput(view) - input, c.sideInput(view) + input), requiresSideInputs(view)))); PAssert.that(output).containsInAnyOrder(38, 39, 40, 40, 41, 42); pipeline.run(); }
.withNaming( Contextful.of( tableManifestNaming, Requirements.requiresSideInputs(outputDirectoryName))) .via(Contextful.fn(KV::getValue), TextIO.sink()) .withTempDirectory(outputDir)); .withNaming( Contextful.of( manifestNaming, Requirements.requiresSideInputs(outputDirectoryName)))); return fileWriteResults;