/** Like {@link #requiresSideInputs(Collection)}. */ public static Requirements requiresSideInputs(PCollectionView<?>... sideInputs) { return requiresSideInputs(Arrays.asList(sideInputs)); }
/** Watches the growth of the given poll function. See class documentation for more details. */ public static <InputT, OutputT> Growth<InputT, OutputT, OutputT> growthOf( Growth.PollFn<InputT, OutputT> pollFn) { return growthOf(pollFn, Requirements.empty()); }
/** Describes the need for access to the given side inputs. */ public static Requirements requiresSideInputs(Collection<PCollectionView<?>> sideInputs) { return new Requirements(sideInputs); }
public static Requirements union(Contextful... contextfuls) { Set<PCollectionView<?>> sideInputs = Sets.newHashSet(); for (Contextful c : contextfuls) { sideInputs.addAll(c.getRequirements().getSideInputs()); } return requiresSideInputs(sideInputs); } }
private Collection<PCollectionView<?>> getAllSideInputs() { return Requirements.union(getDestinationFn(), getOutputFn(), getSinkFn(), getFileNamingFn()) .getSideInputs(); }
.withSideInputs(fn.getRequirements().getSideInputs()));
.apply( ParDo.of(new WatchGrowthFn<>(this, outputCoder, outputKeyFn, outputKeyCoder)) .withSideInputs(getPollFn().getRequirements().getSideInputs())) .setCoder(KvCoder.of(input.getCoder(), outputCoder));
@Override public PCollection<T> expand(PCollection<T> input) { List<PCollectionView<?>> views = Lists.newArrayList(); for (int i = 0; i < signals.size(); ++i) { views.add(signals.get(i).apply("To wait view " + i, new ToWaitView())); } return input.apply( "Wait", MapElements.into(input.getCoder().getEncodedTypeDescriptor()) .via(fn((t, c) -> t, requiresSideInputs(views)))); } }
/** * Wraps a {@link SerializableFunction} as a {@link Contextful} of {@link Fn} with empty {@link * Requirements}. */ public static <InputT, OutputT> Contextful<Fn<InputT, OutputT>> fn( final SerializableFunction<InputT, OutputT> fn) { return new Contextful<>((element, c) -> fn.apply(element), Requirements.empty()); }
.withSideInputs(fn.getRequirements().getSideInputs()));
/** Describes an empty set of requirements. */ public static Requirements empty() { return new Requirements(Collections.emptyList()); }
requiresSideInputs(schemaView))) .via( fn( return AvroIO.sinkViaGenericRecords(schema, formatter); }, requiresSideInputs(schemaView))) .to(baseDir.toString()) .withNaming( requiresSideInputs(schemaView))) .withTempDirectory(baseDir.toString()) .withDestinationCoder(StringUtf8Coder.of())
@Override public PCollection<MatchResult.Metadata> expand(PCollection<String> input) { PCollection<MatchResult.Metadata> res; if (getConfiguration().getWatchInterval() == null) { res = input.apply( "Match filepatterns", ParDo.of(new MatchFn(getConfiguration().getEmptyMatchTreatment()))); } else { res = input .apply( "Continuously match filepatterns", Watch.growthOf( Contextful.of(new MatchPollFn(), Requirements.empty()), new ExtractFilenameFn()) .withPollInterval(getConfiguration().getWatchInterval()) .withTerminationPerInput(getConfiguration().getWatchTerminationCondition())) .apply(Values.create()); } return res.apply(Reshuffle.viaRandomKey()); }
@Test @Category(NeedsRunner.class) public void testSinglePollMultipleInputsWithSideInput() { final PCollectionView<String> moo = p.apply("moo", Create.of("moo")).apply("moo singleton", View.asSingleton()); final PCollectionView<String> zoo = p.apply("zoo", Create.of("zoo")).apply("zoo singleton", View.asSingleton()); PCollection<KV<String, String>> res = p.apply("input", Create.of("a", "b")) .apply( Watch.growthOf( new PollFn<String, String>() { @Override public PollResult<String> apply(String element, Context c) throws Exception { return PollResult.complete( Instant.now(), Arrays.asList( element + " " + c.sideInput(moo) + " " + c.sideInput(zoo))); } }, requiresSideInputs(moo, zoo)) .withPollInterval(Duration.ZERO)); PAssert.that(res) .containsInAnyOrder(Arrays.asList(KV.of("a", "a moo zoo"), KV.of("b", "b moo zoo"))); p.run(); }
standardSeconds(3) /* timeToDeclareOutputFinal */, standardSeconds(30) /* timeToFail */), Requirements.empty()), KV::getKey) .withTerminationPerInput(Growth.afterTotalOf(standardSeconds(5)))
/** Basic test of {@link MapElements} with a {@link Fn} and a side input. */ @Test @Category(NeedsRunner.class) public void testMapBasicWithSideInput() throws Exception { final PCollectionView<Integer> view = pipeline.apply("Create base", Create.of(40)).apply(View.asSingleton()); PCollection<Integer> output = pipeline .apply(Create.of(0, 1, 2)) .apply( MapElements.into(integers()) .via( fn((element, c) -> element + c.sideInput(view), requiresSideInputs(view)))); PAssert.that(output).containsInAnyOrder(40, 41, 42); pipeline.run(); }
/** Basic test of {@link FlatMapElements} with a {@link Fn} and a side input. */ @Test @Category(NeedsRunner.class) public void testFlatMapBasicWithSideInput() throws Exception { final PCollectionView<Integer> view = pipeline.apply("Create base", Create.of(40)).apply(View.asSingleton()); PCollection<Integer> output = pipeline .apply(Create.of(0, 1, 2)) .apply( FlatMapElements.into(integers()) .via( fn( (input, c) -> ImmutableList.of( c.sideInput(view) - input, c.sideInput(view) + input), requiresSideInputs(view)))); PAssert.that(output).containsInAnyOrder(38, 39, 40, 40, 41, 42); pipeline.run(); }
.withNaming( Contextful.of( tableManifestNaming, Requirements.requiresSideInputs(outputDirectoryName))) .via(Contextful.fn(KV::getValue), TextIO.sink()) .withTempDirectory(outputDir)); .withNaming( Contextful.of( manifestNaming, Requirements.requiresSideInputs(outputDirectoryName)))); return fileWriteResults;