standardSeconds(30) /* timeToFail */)) .withPollInterval(Duration.millis(500)) .withOutputCoder(VarIntCoder.of())) .apply(Reify.timestampsInValue()) .apply("Drop timestamped input", Values.create());
.withTerminationPerInput(Growth.afterTotalOf(standardSeconds(1))) .withPollInterval(Duration.millis(1)) .withOutputCoder(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of()))) .apply("Drop input", Values.create());
private void testMultiplePolls(boolean terminationConditionElapsesBeforeOutputIsFinal) { List<Integer> all = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); PCollection<Integer> res = p.apply(Create.of("a")) .apply( Watch.growthOf( new TimedPollFn<String, Integer>( all, standardSeconds(1) /* timeToOutputEverything */, standardSeconds(3) /* timeToDeclareOutputFinal */, standardSeconds(30) /* timeToFail */)) .withTerminationPerInput( Growth.afterTotalOf( standardSeconds( // At 2 seconds, all output has been yielded, but not yet // declared final - so polling should terminate per termination // condition. // At 3 seconds, all output has been yielded (and declared final), // so polling should terminate because of that without waiting for // 100 seconds. terminationConditionElapsesBeforeOutputIsFinal ? 2 : 100))) .withPollInterval(Duration.millis(300)) .withOutputCoder(VarIntCoder.of())) .apply("Drop input", Values.create()); PAssert.that(res).containsInAnyOrder(all); p.run(); }
.withTerminationPerInput(Growth.afterTotalOf(standardSeconds(5))) .withPollInterval(Duration.millis(100)) .withOutputCoder(KvCoder.of(VarIntCoder.of(), StringUtf8Coder.of())) .withOutputKeyCoder(VarIntCoder.of())) .apply("Drop input", Values.create())
@Test @Category(NeedsRunner.class) public void testMultiplePollsStopAfterTimeSinceNewOutput() { List<Integer> all = Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9); PCollection<Integer> res = p.apply(Create.of("a")) .apply( Watch.growthOf( new TimedPollFn<String, Integer>( all, standardSeconds(1) /* timeToOutputEverything */, // Never declare output final standardSeconds(1000) /* timeToDeclareOutputFinal */, standardSeconds(30) /* timeToFail */)) // Should terminate after 4 seconds - earlier than timeToFail .withTerminationPerInput(afterTimeSinceNewOutput(standardSeconds(3))) .withPollInterval(Duration.millis(300)) .withOutputCoder(VarIntCoder.of())) .apply("Drop input", Values.create()); PAssert.that(res).containsInAnyOrder(all); p.run(); }