TextIO.read() .from(options.getInputFilePattern()) .watchForNewFiles(DEFAULT_POLL_INTERVAL, Growth.never()))
/** * Executes the pipeline with the provided execution * parameters. * * @param options The execution parameters. */ public static PipelineResult run(Options options) { // Create the pipeline. Pipeline pipeline = Pipeline.create(options); /* * Steps: * 1) Read from the text source. * 2) Write each text record to Pub/Sub */ pipeline .apply( "Read Text Data", TextIO.read() .from(options.getInputFilePattern()) .watchForNewFiles(DEFAULT_POLL_INTERVAL, Watch.Growth.never())) .apply("Write to PubSub", PubsubIO.writeStrings().to(options.getOutputTopic())); return pipeline.run(); } }
/** * Watches the growth of the given poll function, using the given "key function" to deduplicate * outputs. For example, if OutputT is a filename + file size, this can be a function that returns * just the filename, so that if the same file is observed multiple times with different sizes, * only the first observation is emitted. * * <p>By default, this is the identity function, i.e. the output is used as its own key. */ public static <InputT, OutputT, KeyT> Growth<InputT, OutputT, KeyT> growthOf( Contextful<Growth.PollFn<InputT, OutputT>> pollFn, SerializableFunction<OutputT, KeyT> outputKeyFn) { checkArgument(pollFn != null, "pollFn can not be null"); checkArgument(outputKeyFn != null, "outputKeyFn can not be null"); return new AutoValue_Watch_Growth.Builder<InputT, OutputT, KeyT>() .setTerminationPerInput(Watch.Growth.never()) .setPollFn(pollFn) .setOutputKeyFn(outputKeyFn) .build(); }
private static GrowthTracker<String, String, Integer> newTracker() { return newTracker(new GrowthState<>(never().forNewInput(Instant.now(), null))); }
private static GrowthTracker<String, String, Integer> newTracker( GrowthState<String, String, Integer> state) { return new GrowthTracker<>( SerializableFunctions.identity(), StringUtf8Coder.of(), state, never()); }
@Test public void testTerminationConditionsNever() { Watch.Growth.Never<Object> c = never(); Integer state = c.forNewInput(Instant.now(), null); assertFalse(c.canStopPolling(Instant.now(), state)); }
/** Watches the growth of the given poll function. See class documentation for more details. */ public static <InputT, OutputT> Growth<InputT, OutputT, OutputT> growthOf( Growth.PollFn<InputT, OutputT> pollFn, Requirements requirements) { return new AutoValue_Watch_Growth.Builder<InputT, OutputT, OutputT>() .setTerminationPerInput(Growth.never()) .setPollFn(Contextful.of(pollFn, requirements)) // use null as a signal that this is the identity function and output coder can be // reused as key coder .setOutputKeyFn(null) .build(); }