ParDoTranslation.class.getSimpleName()); TupleTag<?> tag = new TupleTag<>(localName); WindowMappingFn<?> windowMappingFn = windowMappingFnFromProto(sideInput.getWindowMappingFn()); ViewFn<?, ?> viewFn = viewFnFromProto(sideInput.getViewFn());
public static <UserT, DestinationT> List<PCollectionView<?>> getDynamicDestinationSideInputs( AppliedPTransform< PCollection<UserT>, WriteFilesResult<DestinationT>, ? extends PTransform<PCollection<UserT>, WriteFilesResult<DestinationT>>> transform) throws IOException { SdkComponents sdkComponents = SdkComponents.create(transform.getPipeline().getOptions()); RunnerApi.PTransform transformProto = PTransformTranslation.toProto(transform, sdkComponents); List<PCollectionView<?>> views = Lists.newArrayList(); Map<String, SideInput> sideInputs = getWriteFilesPayload(transform).getSideInputsMap(); for (Map.Entry<String, SideInput> entry : sideInputs.entrySet()) { PCollection<?> originalPCollection = checkNotNull( (PCollection<?>) transform.getInputs().get(new TupleTag<>(entry.getKey())), "no input with tag %s", entry.getKey()); views.add( PCollectionViewTranslation.viewFromProto( entry.getValue(), entry.getKey(), originalPCollection, transformProto, RehydratedComponents.forComponents(sdkComponents.toComponents()))); } return views; }
@Test public void testViewFnTranslation() throws Exception { SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); assertEquals( new TestViewFn(), PCollectionViewTranslation.viewFnFromProto( ParDoTranslation.translateViewFn(new TestViewFn(), sdkComponents))); }
@Test public void testWindowMappingFnTranslation() throws Exception { SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); assertEquals( new GlobalWindows().getDefaultWindowMappingFn(), PCollectionViewTranslation.windowMappingFnFromProto( ParDoTranslation.translateWindowMappingFn( new GlobalWindows().getDefaultWindowMappingFn(), sdkComponents))); }
public static List<PCollectionView<?>> getSideInputs(AppliedPTransform<?, ?, ?> application) throws IOException { PTransform<?, ?> transform = application.getTransform(); if (transform instanceof ParDo.MultiOutput) { return ((ParDo.MultiOutput<?, ?>) transform).getSideInputs(); } SdkComponents sdkComponents = SdkComponents.create(application.getPipeline().getOptions()); RunnerApi.PTransform parDoProto = PTransformTranslation.toProto(application, sdkComponents); ParDoPayload payload = ParDoPayload.parseFrom(parDoProto.getSpec().getPayload()); List<PCollectionView<?>> views = new ArrayList<>(); RehydratedComponents components = RehydratedComponents.forComponents(sdkComponents.toComponents()); for (Map.Entry<String, SideInput> sideInputEntry : payload.getSideInputsMap().entrySet()) { String sideInputTag = sideInputEntry.getKey(); RunnerApi.SideInput sideInput = sideInputEntry.getValue(); PCollection<?> originalPCollection = checkNotNull( (PCollection<?>) application.getInputs().get(new TupleTag<>(sideInputTag)), "no input with tag %s", sideInputTag); views.add( PCollectionViewTranslation.viewFromProto( sideInput, sideInputTag, originalPCollection, parDoProto, components)); } return views; }
SideInput sideInput = parDoPayload.getSideInputsOrThrow(view.getTagInternal().getId()); PCollectionView<?> restoredView = PCollectionViewTranslation.viewFromProto( sideInput, view.getTagInternal().getId(),