@Test public void parDoSplittable() { AppliedPTransform<?, ?, ?> parDoApplication = getAppliedTransform( ParDo.of(splittableDoFn).withOutputTags(new TupleTag<>(), TupleTagList.empty())); assertThat(PTransformMatchers.splittableParDo().matches(parDoApplication), is(true)); assertThat(PTransformMatchers.stateOrTimerParDoMulti().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.splittableParDoSingle().matches(parDoApplication), is(false)); assertThat(PTransformMatchers.stateOrTimerParDoSingle().matches(parDoApplication), is(false)); }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); // TODO: [BEAM-5358] Support @RequiresStableInput on Spark runner builder.add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Spark runner currently doesn't support @RequiresStableInput annotation."))); if (!streaming) { builder .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), new SplittableParDoNaiveBounded.OverrideFactory())); } return builder.build(); } }
public static List<PTransformOverride> getDefaultOverrides(boolean streaming) { ImmutableList.Builder<PTransformOverride> builder = ImmutableList.builder(); builder // TODO: [BEAM-5359] Support @RequiresStableInput on Flink runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Flink runner currently doesn't support @RequiresStableInput annotation."))) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.SPLITTABLE_PROCESS_KEYED_URN), streaming ? new SplittableParDoViaKeyedWorkItems.OverrideFactory() : new SplittableParDoNaiveBounded.OverrideFactory())); if (streaming) { builder.add( PTransformOverride.of( PTransformMatchers.urnEqualTo(PTransformTranslation.CREATE_VIEW_TRANSFORM_URN), new CreateStreamingFlinkView.Factory())); } return builder.build(); } }
public void translate(Pipeline pipeline) { List<PTransformOverride> overrides = ImmutableList.<PTransformOverride>builder() .add( PTransformOverride.of( PTransformMatchers.classEqualTo(View.CreatePCollectionView.class), new CreateStreamingGearpumpView.Factory())) .add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of( PTransformMatchers.splittableProcessKeyedBounded(), new SplittableParDoNaiveBounded.OverrideFactory())) // TODO: [BEAM-5361] Support @RequiresStableInput on Gearpump runner .add( PTransformOverride.of( PTransformMatchers.requiresStableInputParDoMulti(), UnsupportedOverrideFactory.withMessage( "Gearpump runner currently doesn't support @RequiresStableInput annotation."))) .build(); pipeline.replaceAll(overrides); pipeline.traverseTopologically(this); }
.add( PTransformOverride.of( PTransformMatchers.splittableParDo(), new SplittableParDo.OverrideFactory())) .add( PTransformOverride.of(