/** Whether the {@link DoFn} described by this signature uses timers. */ public boolean usesTimers() { return timerDeclarations().size() > 0; }
@Override public Map<String, RunnerApi.TimerSpec> translateTimerSpecs( SdkComponents newComponents) { Map<String, RunnerApi.TimerSpec> timerSpecs = new HashMap<>(); for (Map.Entry<String, DoFnSignature.TimerDeclaration> timer : signature.timerDeclarations().entrySet()) { RunnerApi.TimerSpec spec = translateTimerSpec(getTimerSpecOrThrow(timer.getValue(), doFn), newComponents); timerSpecs.put(timer.getKey(), spec); } return timerSpecs; }
@Override public Map<String, RunnerApi.TimerSpec> translateTimerSpecs(SdkComponents newComponents) { Map<String, RunnerApi.TimerSpec> timerSpecs = new HashMap<>(); for (Map.Entry<String, TimerDeclaration> timer : signature.timerDeclarations().entrySet()) { RunnerApi.TimerSpec spec = translateTimerSpec(getTimerSpecOrThrow(timer.getValue(), doFn), newComponents); timerSpecs.put(timer.getKey(), spec); } return timerSpecs; }
/** * Reject state and timers {@link DoFn}. * * @param doFn the {@link DoFn} to possibly reject. */ public static void rejectStateAndTimers(DoFn<?, ?> doFn) { DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0) { throw new UnsupportedOperationException( String.format( "Found %s annotations on %s, but %s cannot yet be used with state in the %s.", DoFn.StateId.class.getSimpleName(), doFn.getClass().getName(), DoFn.class.getSimpleName(), SparkRunner.class.getSimpleName())); } if (signature.timerDeclarations().size() > 0) { throw new UnsupportedOperationException( String.format( "Found %s annotations on %s, but %s cannot yet be used with timers in the %s.", DoFn.TimerId.class.getSimpleName(), doFn.getClass().getName(), DoFn.class.getSimpleName(), SparkRunner.class.getSimpleName())); } }
/** * Perform common validations of the {@link DoFn}, for example ensuring that state is used * correctly and that its features can be supported. */ private static <InputT, OutputT> void validate(DoFn<InputT, OutputT> fn) { DoFnSignature signature = DoFnSignatures.getSignature((Class) fn.getClass()); // State is semantically incompatible with splitting if (!signature.stateDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses state, but these are not compatible", fn.getClass().getName())); } // Timers are semantically incompatible with splitting if (!signature.timerDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses timers, but these are not compatible", fn.getClass().getName())); } }
if (signature.timerDeclarations().size() > 0) { throw new UnsupportedOperationException( String.format(
@Test public void testSimpleTimerIdNamedDoFn() throws Exception { class DoFnForTestSimpleTimerIdNamedDoFn extends DoFn<KV<String, Integer>, Long> { @TimerId("foo") private final TimerSpec bizzle = TimerSpecs.timer(TimeDomain.EVENT_TIME); @ProcessElement public void foo(ProcessContext context) {} @OnTimer("foo") public void onFoo() {} } // Test classes at the bottom of the file DoFnSignature sig = DoFnSignatures.signatureForDoFn(new DoFnForTestSimpleTimerIdNamedDoFn()); assertThat(sig.timerDeclarations().size(), equalTo(1)); DoFnSignature.TimerDeclaration decl = sig.timerDeclarations().get("foo"); assertThat(decl.id(), equalTo("foo")); assertThat( decl.field(), equalTo(DoFnForTestSimpleTimerIdNamedDoFn.class.getDeclaredField("bizzle"))); }
@Test public void testSimpleTimerIdAnonymousDoFn() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<KV<String, Integer>, Long>() { @TimerId("foo") private final TimerSpec bizzle = TimerSpecs.timer(TimeDomain.EVENT_TIME); @ProcessElement public void foo(ProcessContext context) {} @OnTimer("foo") public void onFoo() {} }.getClass()); assertThat(sig.timerDeclarations().size(), equalTo(1)); DoFnSignature.TimerDeclaration decl = sig.timerDeclarations().get("foo"); assertThat(decl.id(), equalTo("foo")); assertThat(decl.field().getName(), equalTo("bizzle")); }
@SuppressWarnings("unchecked") private PTransform<PCollection<? extends InputT>, PCollectionTuple> getReplacementForApplication( AppliedPTransform< PCollection<? extends InputT>, PCollectionTuple, PTransform<PCollection<? extends InputT>, PCollectionTuple>> application) throws IOException { DoFn<InputT, OutputT> fn = (DoFn<InputT, OutputT>) ParDoTranslation.getDoFn(application); DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); if (signature.processElement().isSplittable()) { return SplittableParDo.forAppliedParDo((AppliedPTransform) application); } else if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) { return new GbkThenStatefulParDo( fn, ParDoTranslation.getMainOutputTag(application), ParDoTranslation.getAdditionalOutputTags(application), ParDoTranslation.getSideInputs(application)); } else { return application.getTransform(); } }
@Test public void testDeclAndUsageOfTimerInSuperclass() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature(new DoFnOverridingAbstractTimerUse().getClass()); assertThat(sig.timerDeclarations().size(), equalTo(1)); assertThat(sig.processElement().extraParameters().size(), equalTo(2)); DoFnSignature.TimerDeclaration decl = sig.timerDeclarations().get(DoFnOverridingAbstractTimerUse.TIMER_ID); TimerParameter timerParam = (TimerParameter) sig.processElement().extraParameters().get(1); assertThat( decl.field(), equalTo(DoFnDeclaringTimerAndAbstractUse.class.getDeclaredField("myTimerSpec"))); // The method we pull out is the superclass method; this is what allows validation to remain // simple. The later invokeDynamic instruction causes it to invoke the actual implementation. assertThat(timerParam.referent(), equalTo(decl)); }
@Test public void testSimpleTimerWithContext() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<KV<String, Integer>, Long>() { @TimerId("foo") private final TimerSpec bizzle = TimerSpecs.timer(TimeDomain.EVENT_TIME); @ProcessElement public void foo(ProcessContext context) {} @OnTimer("foo") public void onFoo(OnTimerContext c) {} }.getClass()); assertThat(sig.timerDeclarations().size(), equalTo(1)); DoFnSignature.TimerDeclaration decl = sig.timerDeclarations().get("foo"); assertThat(decl.id(), equalTo("foo")); assertThat(decl.field().getName(), equalTo("bizzle")); assertThat( sig.onTimerMethods().get("foo").extraParameters().get(0), equalTo((Parameter) Parameter.onTimerContext())); }
/** * In this particular test, the super class annotated both the timer and the callback, and the * subclass overrides an abstract method. This is allowed. */ @Test public void testOnTimerDeclaredAndUsedInSuperclass() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature(new DoFnOverridingAbstractCallback().getClass()); assertThat(sig.timerDeclarations().size(), equalTo(1)); assertThat(sig.onTimerMethods().size(), equalTo(1)); DoFnSignature.TimerDeclaration decl = sig.timerDeclarations().get(DoFnDeclaringTimerAndAbstractCallback.TIMER_ID); DoFnSignature.OnTimerMethod callback = sig.onTimerMethods().get(DoFnDeclaringTimerAndAbstractCallback.TIMER_ID); assertThat( decl.field(), equalTo(DoFnDeclaringTimerAndAbstractCallback.class.getDeclaredField("myTimerSpec"))); // The method we pull out is the superclass method; this is what allows validation to remain // simple. The later invokeDynamic instruction causes it to invoke the actual implementation. assertThat( callback.targetMethod(), equalTo(DoFnDeclaringTimerAndAbstractCallback.class.getDeclaredMethod("onMyTimer"))); }
boolean stateful = false; DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
boolean stateful = false; DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
DoFnSignature signature = DoFnSignatures.getSignature(transform.getFn().getClass()); if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
for (String localTimerName : signature.timerDeclarations().keySet()) { RunnerApi.PCollection timerPCollection = components.getPcollectionsOrThrow(String.format("foo.%s", localTimerName));