public StateInternalsStateCleaner( DoFn<?, ?> fn, StateInternals stateInternals, Coder<W> windowCoder) { this.fn = fn; this.signature = DoFnSignatures.getSignature(fn.getClass()); this.stateInternals = stateInternals; this.windowCoder = windowCoder; }
private static boolean isSplittable(DoFn<?, ?> fn) { return DoFnSignatures.signatureForDoFn(fn).processElement().isSplittable(); } }
fnContext.addStateDeclarations(analyzeStateDeclarations(errors, fnClass).values()); fnContext.addTimerDeclarations(analyzeTimerDeclarations(errors, fnClass).values()); fnContext.addFieldAccessDeclarations(analyzeFieldAccessDeclaration(errors, fnClass).values()); findAnnotatedMethod(errors, DoFn.ProcessElement.class, fnClass, true); Method startBundleMethod = findAnnotatedMethod(errors, DoFn.StartBundle.class, fnClass, false); Method finishBundleMethod = findAnnotatedMethod(errors, DoFn.FinishBundle.class, fnClass, false); Method setupMethod = findAnnotatedMethod(errors, DoFn.Setup.class, fnClass, false); Method teardownMethod = findAnnotatedMethod(errors, DoFn.Teardown.class, fnClass, false); Method onWindowExpirationMethod = findAnnotatedMethod(errors, DoFn.OnWindowExpiration.class, fnClass, false); Method getInitialRestrictionMethod = findAnnotatedMethod(errors, DoFn.GetInitialRestriction.class, fnClass, false); Method splitRestrictionMethod = findAnnotatedMethod(errors, DoFn.SplitRestriction.class, fnClass, false); Method getRestrictionCoderMethod = findAnnotatedMethod(errors, DoFn.GetRestrictionCoder.class, fnClass, false); Method newTrackerMethod = findAnnotatedMethod(errors, DoFn.NewTracker.class, fnClass, false); declaredMethodsWithAnnotation(DoFn.OnTimer.class, fnClass, DoFn.class); HashMap<String, DoFnSignature.OnTimerMethod> onTimerMethodMap = Maps.newHashMapWithExpectedSize(onTimerMethods.size()); id, analyzeOnTimerMethod(errors, fnT, onTimerMethod, id, inputT, outputT, fnContext)); errors.forMethod(DoFn.ProcessElement.class, processElementMethod); DoFnSignature.ProcessElementMethod processElement = analyzeProcessElementMethod(
@Test public void testSplitRestrictionReturnsWrongType() throws Exception { thrown.expectMessage( "Third argument must be DoFn.OutputReceiver<SomeRestriction>, " + "but is DoFn.OutputReceiver<String>"); DoFnSignatures.analyzeSplitRestrictionMethod( errors(), TypeDescriptor.of(FakeDoFn.class), new AnonymousMethod() { void method( Integer element, SomeRestriction restriction, DoFn.OutputReceiver<String> receiver) {} }.getMethod(), TypeDescriptor.of(Integer.class)); }
@Test public void testNewTrackerWrongNumArguments() throws Exception { thrown.expectMessage("Must have a single argument"); DoFnSignatures.analyzeNewTrackerMethod( errors(), TypeDescriptor.of(FakeDoFn.class), new AnonymousMethod() { private SomeRestrictionTracker method(SomeRestriction restriction, Object extra) { return null; } }.getMethod()); }
static DoFnSignature.ProcessElementMethod analyzeProcessElementMethod(AnonymousMethod method) throws Exception { return DoFnSignatures.analyzeProcessElementMethod( errors(), TypeDescriptor.of(FakeDoFn.class), method.getMethod(), TypeDescriptor.of(Integer.class), TypeDescriptor.of(String.class), FnAnalysisContext.create()); } }
@Test public void testBadExtraContext() throws Exception { thrown.expect(IllegalArgumentException.class); thrown.expectMessage( "Must take a single argument of type DoFn<Integer, String>.StartBundleContext"); DoFnSignatures.analyzeStartBundleMethod( errors(), TypeDescriptor.of(FakeDoFn.class), new DoFnSignaturesTestUtils.AnonymousMethod() { void method(DoFn<Integer, String>.StartBundleContext c, int n) {} }.getMethod(), TypeDescriptor.of(Integer.class), TypeDescriptor.of(String.class)); }
@Test public void testSplitRestrictionWrongNumArguments() throws Exception { thrown.expectMessage("Must have exactly 3 arguments"); DoFnSignatures.analyzeSplitRestrictionMethod( errors(), TypeDescriptor.of(FakeDoFn.class), new AnonymousMethod() { private void method( Integer element, SomeRestriction restriction, DoFn.OutputReceiver<SomeRestriction> receiver, Object extra) {} }.getMethod(), TypeDescriptor.of(Integer.class)); }
@Test public void testNewTrackerInconsistent() throws Exception { thrown.expectMessage( "Returns SomeRestrictionTracker, " + "but must return a subtype of RestrictionTracker<String, ?>"); DoFnSignatures.analyzeNewTrackerMethod( errors(), TypeDescriptor.of(FakeDoFn.class), new AnonymousMethod() { private SomeRestrictionTracker method(String restriction) { return null; } }.getMethod()); } }
/** @return the {@link DoFnSignature} for the given {@link DoFn} instance. */ public static <FnT extends DoFn<?, ?>> DoFnSignature signatureForDoFn(FnT fn) { return getSignature(fn.getClass()); }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.SingleOutput) { DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().requiresStableInput(); } return false; }
@Test public void testSplitRestrictionWrongElementArgument() throws Exception { class BadFn { private List<SomeRestriction> splitRestriction(String element, SomeRestriction restriction) { return null; } } thrown.expectMessage("First argument must be the element type Integer"); DoFnSignatures.analyzeSplitRestrictionMethod( errors(), TypeDescriptor.of(FakeDoFn.class), new AnonymousMethod() { void method( String element, SomeRestriction restriction, DoFn.OutputReceiver<SomeRestriction> receiver) {} }.getMethod(), TypeDescriptor.of(Integer.class)); }
/** @return the {@link DoFnInvoker} for the given {@link DoFn}. */ @SuppressWarnings({"unchecked", "rawtypes"}) public <InputT, OutputT> DoFnInvoker<InputT, OutputT> newByteBuddyInvoker( DoFn<InputT, OutputT> fn) { return newByteBuddyInvoker(DoFnSignatures.getSignature((Class) fn.getClass()), fn); }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.SingleOutput) { DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.usesState() || signature.usesTimers(); } return false; }
private static void rejectSplittable(DoFn<?, ?> doFn) { DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s does not currently support splittable DoFn: %s", FlinkRunner.class.getSimpleName(), doFn)); } }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.usesState() || signature.usesTimers(); } return false; }
private static <InputT, OutputT> void verifyFnIsStateful(DoFn<InputT, OutputT> fn) { DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); // It is still correct to use this without state or timers, but a bad idea. // Since it is internal it should never be used wrong, so it is OK to crash. checkState( signature.usesState() || signature.usesTimers(), "%s used for %s that does not use state or timers.", BatchStatefulParDoOverrides.class.getSimpleName(), ParDo.class.getSimpleName()); } }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().requiresStableInput(); } return false; }
@Override public Constructor<?> load(final OnTimerMethodSpecifier onTimerMethodSpecifier) throws Exception { DoFnSignature signature = DoFnSignatures.getSignature(onTimerMethodSpecifier.fnClass()); Class<? extends OnTimerInvoker<?, ?>> invokerClass = generateOnTimerInvokerClass(signature, onTimerMethodSpecifier.timerId()); try { return invokerClass.getConstructor(signature.fnClass()); } catch (IllegalArgumentException | NoSuchMethodException | SecurityException e) { throw new RuntimeException(e); } } });
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable(); } return false; }