@Override public boolean isSplittable() { return signature.processElement().isSplittable(); }
private static void rejectSplittable(DoFn<?, ?> doFn) { DoFnSignature signature = DoFnSignatures.getSignature(doFn.getClass()); if (signature.processElement().isSplittable()) { throw new UnsupportedOperationException( String.format( "%s does not currently support splittable DoFn: %s", FlinkRunner.class.getSimpleName(), doFn)); } }
@Override public List<RunnerApi.Parameter> translateParameters() { return ParDoTranslation.translateParameters( signature.processElement().extraParameters()); }
@Override public List<Parameter> translateParameters() { return ParDoTranslation.translateParameters( signature.processElement().extraParameters()); }
private static boolean isSplittable(DoFn<?, ?> fn) { return DoFnSignatures.signatureForDoFn(fn).processElement().isSplittable(); } }
@Override public List<RunnerApi.Parameter> translateParameters() { return ParDoTranslation.translateParameters( signature.processElement().extraParameters()); }
private SplittableParDo( DoFn<InputT, OutputT> doFn, List<PCollectionView<?>> sideInputs, TupleTag<OutputT> mainOutputTag, TupleTagList additionalOutputTags, Map<TupleTag<?>, Coder<?>> outputTagsToCoders) { checkArgument( DoFnSignatures.getSignature(doFn.getClass()).processElement().isSplittable(), "fn must be a splittable DoFn"); this.doFn = doFn; this.sideInputs = sideInputs; this.mainOutputTag = mainOutputTag; this.additionalOutputTags = additionalOutputTags; this.outputTagsToCoders = outputTagsToCoders; }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.SingleOutput) { DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().requiresStableInput(); } return false; }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().requiresStableInput(); } return false; }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { DoFn<?, ?> fn = ((ParDo.MultiOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable(); } return false; }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.SingleOutput) { DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable(); } return false; }
@Test public void testBasicDoFnMultiOutputReceiver() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<String, String>() { @ProcessElement public void process(MultiOutputReceiver receiver) {} }.getClass()); assertThat(sig.processElement().extraParameters().size(), equalTo(1)); assertThat( sig.processElement().extraParameters().get(0), instanceOf(TaggedOutputReceiverParameter.class)); }
@Test public void testBasicDoFnProcessContext() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<String, String>() { @ProcessElement public void process(ProcessContext c) {} }.getClass()); assertThat(sig.processElement().extraParameters().size(), equalTo(1)); assertThat( sig.processElement().extraParameters().get(0), instanceOf(ProcessContextParameter.class)); }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof SplittableParDo.ProcessKeyedElements) { DoFn<?, ?> fn = ((SplittableParDo.ProcessKeyedElements) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable() && signature.isBoundedPerElement() == IsBounded.BOUNDED; } return false; }
@Override public boolean matches(AppliedPTransform<?, ?, ?> application) { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof SplittableParDo.ProcessKeyedElements) { DoFn<?, ?> fn = ((SplittableParDo.ProcessKeyedElements) transform).getFn(); DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn); return signature.processElement().isSplittable() && signature.isBoundedPerElement() == IsBounded.UNBOUNDED; } return false; }
@Test public void testPipelineOptionsParameter() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<String, String>() { @ProcessElement public void process(ProcessContext c, PipelineOptions options) {} }.getClass()); assertThat( sig.processElement().extraParameters(), Matchers.hasItem(instanceOf(Parameter.PipelineOptionsParameter.class))); }
@Test public void testRequiresStableInputProcessElement() throws Exception { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<String, String>() { @ProcessElement @RequiresStableInput public void process(ProcessContext c) {} }.getClass()); assertThat(sig.processElement().requiresStableInput(), is(true)); }
@Test public void testRowParameterWithoutFieldAccess() { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<String, String>() { @ProcessElement public void process(@Element Row row) {} }.getClass()); assertThat(sig.processElement().getRowParameter(), notNullValue()); }
@Test public void testRowReceiver() { DoFnSignature sig = DoFnSignatures.getSignature( new DoFn<String, String>() { @ProcessElement public void process(OutputReceiver<Row> rowReceiver) {} }.getClass()); assertThat(sig.processElement().getMainOutputReceiver().isRowReceiver(), is(true)); }
private static Implementation getRestrictionCoderDelegation( TypeDescription doFnType, DoFnSignature signature) { if (signature.processElement().isSplittable()) { if (signature.getRestrictionCoder() == null) { return MethodDelegation.to( new DefaultRestrictionCoder(signature.getInitialRestriction().restrictionT())); } else { return new DowncastingParametersMethodDelegation( doFnType, signature.getRestrictionCoder().targetMethod()); } } else { return ExceptionMethod.throwing(UnsupportedOperationException.class); } }