@Override public boolean hasTranslation(PTransform<?, ?> transform) { // streaming includes rdd/bounded transformations as well return EVALUATORS.containsKey(PTransformTranslation.urnForTransformOrNull(transform)); }
/** * Translates a composite {@link AppliedPTransform} into a runner API proto with no component * transforms. * * <p>This should not be used when translating a {@link Pipeline}. * * <p>Does not register the {@code appliedPTransform} within the provided {@link SdkComponents}. */ static RunnerApi.PTransform toProto( AppliedPTransform<?, ?, ?> appliedPTransform, SdkComponents components) throws IOException { return toProto(appliedPTransform, Collections.emptyList(), components); }
@Override public RunnerApi.PTransform translate( AppliedPTransform<?, ?, ?> appliedPTransform, List<AppliedPTransform<?, ?, ?>> subtransforms, SdkComponents components) throws IOException { return translateAppliedPTransform(appliedPTransform, subtransforms, components).build(); } }
@Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { LOG.info("{} visitPrimitiveTransform- {}", genSpaces(this.depth), node.getFullName()); // get the transformation corresponding to the node we are // currently visiting and translate it into its Flink alternative. PTransform<?, ?> transform = node.getTransform(); BatchTransformTranslator<?> translator = FlinkBatchTransformTranslators.getTranslator(transform); if (translator == null) { String transformUrn = PTransformTranslation.urnForTransform(transform); throw new UnsupportedOperationException( "The transform " + transformUrn + " is currently not supported."); } applyBatchTransform(transform, node, translator); }
@Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { LOG.info("{} visitPrimitiveTransform- {}", genSpaces(this.depth), node.getFullName()); // get the transformation corresponding to the node we are // currently visiting and translate it into its Flink alternative. PTransform<?, ?> transform = node.getTransform(); BatchTransformTranslator<?> translator = FlinkBatchTransformTranslators.getTranslator(transform); if (translator == null) { String transformUrn = PTransformTranslation.urnForTransform(transform); throw new UnsupportedOperationException( "The transform " + transformUrn + " is currently not supported."); } applyBatchTransform(transform, node, translator); }
@Override public boolean test(RunnerApi.PTransform pTransform) { return STREAMING_IMPULSE_TRANSFORM_URN.equals( PTransformTranslation.urnForTransformOrNull(pTransform)); } }
private static <T, DestinationT> WriteFilesPayload getWriteFilesPayload( AppliedPTransform< PCollection<T>, WriteFilesResult<DestinationT>, ? extends PTransform<PCollection<T>, WriteFilesResult<DestinationT>>> transform) throws IOException { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); return WriteFilesPayload.parseFrom( PTransformTranslation.toProto(transform, Collections.emptyList(), components) .getSpec() .getPayload()); }
@Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { LOG.info("{} visitPrimitiveTransform- {}", genSpaces(this.depth), node.getFullName()); // get the transformation corresponding to hte node we are // currently visiting and translate it into its Flink alternative. PTransform<?, ?> transform = node.getTransform(); StreamTransformTranslator<?> translator = FlinkStreamingTransformTranslators.getTranslator(transform); if (translator == null || !applyCanTranslate(transform, node, translator)) { String transformUrn = PTransformTranslation.urnForTransform(transform); LOG.info(transformUrn); throw new UnsupportedOperationException( "The transform " + transformUrn + " is currently not supported."); } applyStreamingTransform(transform, node, translator); }
@Override public RunnerApi.PTransform translate( AppliedPTransform<?, ?, ?> appliedPTransform, List<AppliedPTransform<?, ?, ?>> subtransforms, SdkComponents components) throws IOException { RunnerApi.PTransform.Builder transformBuilder = translateAppliedPTransform(appliedPTransform, subtransforms, components); FunctionSpec spec = KNOWN_PAYLOAD_TRANSLATORS .get(appliedPTransform.getTransform().getClass()) .translate(appliedPTransform, components); if (spec != null) { transformBuilder.setSpec(spec); } return transformBuilder.build(); } }
@Override public boolean test(RunnerApi.PTransform pTransform) { return PTransformTranslation.RESHUFFLE_URN.equals( PTransformTranslation.urnForTransformOrNull(pTransform)); } }
private static <T> ReadPayload getReadPayload( AppliedPTransform<PBegin, PCollection<T>, PTransform<PBegin, PCollection<T>>> transform) throws IOException { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); return ReadPayload.parseFrom( PTransformTranslation.toProto(transform, Collections.emptyList(), components) .getSpec() .getPayload()); }
@Override public void visitPrimitiveTransform(TransformHierarchy.Node node) { LOG.info("{} visitPrimitiveTransform- {}", genSpaces(this.depth), node.getFullName()); // get the transformation corresponding to hte node we are // currently visiting and translate it into its Flink alternative. PTransform<?, ?> transform = node.getTransform(); StreamTransformTranslator<?> translator = FlinkStreamingTransformTranslators.getTranslator(transform); if (translator == null || !applyCanTranslate(transform, node, translator)) { String transformUrn = PTransformTranslation.urnForTransform(transform); LOG.info(transformUrn); throw new UnsupportedOperationException( "The transform " + transformUrn + " is currently not supported."); } applyStreamingTransform(transform, node, translator); }
@Override public RunnerApi.PTransform translate( AppliedPTransform<?, ?, ?> appliedPTransform, List<AppliedPTransform<?, ?, ?>> subtransforms, SdkComponents components) throws IOException { RunnerApi.PTransform.Builder transformBuilder = translateAppliedPTransform(appliedPTransform, subtransforms, components); PTransform<?, ?> transform = appliedPTransform.getTransform(); // The raw transform was parsed in the context of other components; this puts it in the // context of our current serialization FunctionSpec spec = ((RawPTransform<?, ?>) transform).migrate(components); // A composite transform is permitted to have a null spec. There are also some pseudo- // primitives not yet supported by the portability framework that have null specs if (spec != null) { transformBuilder.setSpec(spec); } return transformBuilder.build(); } }
@Override public boolean test(RunnerApi.PTransform pTransform) { return STREAMING_IMPULSE_TRANSFORM_URN.equals( PTransformTranslation.urnForTransformOrNull(pTransform)); } }
private static <T> ParDoPayload getParDoPayload(AppliedPTransform<?, ?, ?> transform) throws IOException { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); RunnerApi.PTransform parDoPTransform = PTransformTranslation.toProto(transform, Collections.emptyList(), components); return ParDoPayload.parseFrom(parDoPTransform.getSpec().getPayload()); }
/** * Tests that the translator is registered so the URN can be retrieved (the only thing you can * meaningfully do with a {@link GroupByKey}). */ @Test public void testUrnRetrievable() throws Exception { assertThat( PTransformTranslation.urnForTransform(GroupByKey.create()), equalTo(GROUP_BY_KEY_TRANSFORM_URN)); } }
throws IOException { RunnerApi.PTransform.Builder builder = PTransformTranslation.translateAppliedPTransform( appliedPTransform, subtransforms, components);
@Override public boolean hasTranslation(PTransform<?, ?> transform) { return EVALUATORS.containsKey(PTransformTranslation.urnForTransformOrNull(transform)); }
public static PCollection.IsBounded sourceIsBounded(AppliedPTransform<?, ?, ?> transform) { try { SdkComponents components = SdkComponents.create(transform.getPipeline().getOptions()); return PCollectionTranslation.fromProto( ReadPayload.parseFrom( PTransformTranslation.toProto(transform, Collections.emptyList(), components) .getSpec() .getPayload()) .getIsBounded()); } catch (IOException e) { throw new RuntimeException("Internal error determining boundedness of Read", e); } }
public static FlinkStreamingPipelineTranslator.StreamTransformTranslator<?> getTranslator( PTransform<?, ?> transform) { @Nullable String urn = PTransformTranslation.urnForTransformOrNull(transform); return urn == null ? null : TRANSLATORS.get(urn); }