/** @deprecated use {@link #usesState()}, it's cleaner */ @Deprecated public boolean isStateful() { return stateDeclarations().size() > 0; }
/** Whether the {@link DoFn} described by this signature uses state. */ public boolean usesState() { return stateDeclarations().size() > 0; }
@Override public Map<String, RunnerApi.StateSpec> translateStateSpecs(SdkComponents components) throws IOException { Map<String, RunnerApi.StateSpec> stateSpecs = new HashMap<>(); for (Map.Entry<String, StateDeclaration> state : signature.stateDeclarations().entrySet()) { RunnerApi.StateSpec spec = translateStateSpec(getStateSpecOrThrow(state.getValue(), doFn), components); stateSpecs.put(state.getKey(), spec); } return stateSpecs; }
@Override public Map<String, RunnerApi.StateSpec> translateStateSpecs(SdkComponents components) throws IOException { Map<String, RunnerApi.StateSpec> stateSpecs = new HashMap<>(); for (Map.Entry<String, DoFnSignature.StateDeclaration> state : signature.stateDeclarations().entrySet()) { RunnerApi.StateSpec spec = ParDoTranslation.translateStateSpec( getStateSpecOrThrow(state.getValue(), doFn), components); stateSpecs.put(state.getKey(), spec); } return stateSpecs; }
/** * Reject state and timers {@link DoFn}. * * @param doFn the {@link DoFn} to possibly reject. */ public static void rejectStateAndTimers(DoFn<?, ?> doFn) { DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0) { throw new UnsupportedOperationException( String.format( "Found %s annotations on %s, but %s cannot yet be used with state in the %s.", DoFn.StateId.class.getSimpleName(), doFn.getClass().getName(), DoFn.class.getSimpleName(), SparkRunner.class.getSimpleName())); } if (signature.timerDeclarations().size() > 0) { throw new UnsupportedOperationException( String.format( "Found %s annotations on %s, but %s cannot yet be used with timers in the %s.", DoFn.TimerId.class.getSimpleName(), doFn.getClass().getName(), DoFn.class.getSimpleName(), SparkRunner.class.getSimpleName())); } }
private static void finishSpecifyingStateSpecs( DoFn<?, ?> fn, CoderRegistry coderRegistry, Coder<?> inputCoder) { DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); Map<String, DoFnSignature.StateDeclaration> stateDeclarations = signature.stateDeclarations(); for (DoFnSignature.StateDeclaration stateDeclaration : stateDeclarations.values()) { try { StateSpec<?> stateSpec = (StateSpec<?>) stateDeclaration.field().get(fn); stateSpec.offerCoders(codersForStateSpecTypes(stateDeclaration, coderRegistry, inputCoder)); stateSpec.finishSpecifying(); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } }
/** * Perform common validations of the {@link DoFn}, for example ensuring that state is used * correctly and that its features can be supported. */ private static <InputT, OutputT> void validate(DoFn<InputT, OutputT> fn) { DoFnSignature signature = DoFnSignatures.getSignature((Class) fn.getClass()); // State is semantically incompatible with splitting if (!signature.stateDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses state, but these are not compatible", fn.getClass().getName())); } // Timers are semantically incompatible with splitting if (!signature.timerDeclarations().isEmpty() && signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s is splittable and uses timers, but these are not compatible", fn.getClass().getName())); } }
static void verifyStateSupported(DoFn<?, ?> fn) { DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); for (DoFnSignature.StateDeclaration stateDecl : signature.stateDeclarations().values()) { // https://issues.apache.org/jira/browse/BEAM-1474 if (stateDecl.stateType().isSubtypeOf(TypeDescriptor.of(MapState.class))) { throw new UnsupportedOperationException( String.format( "%s does not currently support %s", DataflowRunner.class.getSimpleName(), MapState.class.getSimpleName())); } // https://issues.apache.org/jira/browse/BEAM-1479 if (stateDecl.stateType().isSubtypeOf(TypeDescriptor.of(SetState.class))) { throw new UnsupportedOperationException( String.format( "%s does not currently support %s", DataflowRunner.class.getSimpleName(), SetState.class.getSimpleName())); } } }
@Override public void clearForWindow(W window) { for (Map.Entry<String, DoFnSignature.StateDeclaration> entry : signature.stateDeclarations().entrySet()) { try { StateSpec<?> spec = (StateSpec<?>) entry.getValue().field().get(fn); State state = stateInternals.state( StateNamespaces.window(windowCoder, window), StateTags.tagForSpec(entry.getKey(), (StateSpec) spec)); state.clear(); } catch (IllegalAccessException e) { throw new RuntimeException(e); } } } }
@SuppressWarnings("unchecked") private PTransform<PCollection<? extends InputT>, PCollectionTuple> getReplacementForApplication( AppliedPTransform< PCollection<? extends InputT>, PCollectionTuple, PTransform<PCollection<? extends InputT>, PCollectionTuple>> application) throws IOException { DoFn<InputT, OutputT> fn = (DoFn<InputT, OutputT>) ParDoTranslation.getDoFn(application); DoFnSignature signature = DoFnSignatures.getSignature(fn.getClass()); if (signature.processElement().isSplittable()) { return SplittableParDo.forAppliedParDo((AppliedPTransform) application); } else if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) { return new GbkThenStatefulParDo( fn, ParDoTranslation.getMainOutputTag(application), ParDoTranslation.getAdditionalOutputTags(application), ParDoTranslation.getSideInputs(application)); } else { return application.getTransform(); } }
@Test public void testSimpleStateIdNamedDoFn() throws Exception { class DoFnForTestSimpleStateIdNamedDoFn extends DoFn<KV<String, Integer>, Long> { @StateId("foo") private final StateSpec<ValueState<Integer>> bizzle = StateSpecs.value(VarIntCoder.of()); @ProcessElement public void foo(ProcessContext context) {} } // Test classes at the bottom of the file DoFnSignature sig = DoFnSignatures.signatureForDoFn(new DoFnForTestSimpleStateIdNamedDoFn()); assertThat(sig.stateDeclarations().size(), equalTo(1)); DoFnSignature.StateDeclaration decl = sig.stateDeclarations().get("foo"); assertThat(decl.id(), equalTo("foo")); assertThat( decl.field(), equalTo(DoFnForTestSimpleStateIdNamedDoFn.class.getDeclaredField("bizzle"))); assertThat( decl.stateType(), Matchers.<TypeDescriptor<?>>equalTo(new TypeDescriptor<ValueState<Integer>>() {})); }
@Test public void testDeclAndUsageOfStateInSuperclass() throws Exception { class DoFnOverridingAbstractStateUse extends DoFnDeclaringStateAndAbstractUse { @Override public void processWithState(ProcessContext c, ValueState<String> state) {} } DoFnSignature sig = DoFnSignatures.getSignature(new DoFnOverridingAbstractStateUse().getClass()); assertThat(sig.stateDeclarations().size(), equalTo(1)); assertThat(sig.processElement().extraParameters().size(), equalTo(2)); DoFnSignature.StateDeclaration decl = sig.stateDeclarations().get(DoFnOverridingAbstractStateUse.STATE_ID); StateParameter stateParam = (StateParameter) sig.processElement().extraParameters().get(1); assertThat( decl.field(), equalTo(DoFnDeclaringStateAndAbstractUse.class.getDeclaredField("myStateSpec"))); // The method we pull out is the superclass method; this is what allows validation to remain // simple. The later invokeDynamic instruction causes it to invoke the actual implementation. assertThat(stateParam.referent(), equalTo(decl)); }
@Test public void testGenericStatefulDoFn() throws Exception { class DoFnForTestGenericStatefulDoFn<T> extends DoFn<KV<String, T>, Long> { // Note that in order to have a coder for T it will require initialization in the constructor, // but that isn't important for this test @StateId("foo") private final StateSpec<ValueState<T>> bizzle = null; @ProcessElement public void foo(ProcessContext context) {} } // Test classes at the bottom of the file DoFn<KV<String, Integer>, Long> myDoFn = new DoFnForTestGenericStatefulDoFn<Integer>() {}; DoFnSignature sig = DoFnSignatures.signatureForDoFn(myDoFn); assertThat(sig.stateDeclarations().size(), equalTo(1)); DoFnSignature.StateDeclaration decl = sig.stateDeclarations().get("foo"); assertThat(decl.id(), equalTo("foo")); assertThat( decl.field(), equalTo(DoFnForTestGenericStatefulDoFn.class.getDeclaredField("bizzle"))); assertThat( decl.stateType(), Matchers.<TypeDescriptor<?>>equalTo(new TypeDescriptor<ValueState<Integer>>() {})); }
@Test public void testSimpleStateIdAnonymousDoFn() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<KV<String, Integer>, Long>() { @StateId("foo") private final StateSpec<ValueState<Integer>> bizzle = StateSpecs.value(VarIntCoder.of()); @ProcessElement public void foo(ProcessContext context) {} }.getClass()); assertThat(sig.stateDeclarations().size(), equalTo(1)); DoFnSignature.StateDeclaration decl = sig.stateDeclarations().get("foo"); assertThat(decl.id(), equalTo("foo")); assertThat(decl.field().getName(), equalTo("bizzle")); assertThat( decl.stateType(), Matchers.<TypeDescriptor<?>>equalTo(new TypeDescriptor<ValueState<Integer>>() {})); }
boolean stateful = false; DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
boolean stateful = false; DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
boolean stateful = false; DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
if (signature.stateDeclarations().size() > 0 || signature.timerDeclarations().size() > 0) {
final DoFnSignature.StateDeclaration decl = sig.stateDeclarations().get("foo"); sig.processElement() .extraParameters()
if (signature.stateDeclarations().size() > 0) { for (final WindowedValue<?> element : inputBundle.getElements()) { for (final BoundedWindow window : element.getWindows()) {