RunnerApi.Components components = pipeline.getComponents(); RunnerApi.PTransform transform = components.getTransformsOrThrow(id); Map<String, String> outputs = transform.getOutputsMap();
Map<String, String> viewOutputsToInputs = new HashMap<>(); pipeline .getComponents() .getTransformsMap() .forEach( .getComponents() .getTransformsMap() .forEach( }); RunnerApi.Pipeline.Builder newPipeline = pipeline.toBuilder(); newPipeline.addAllRootTransformIds( pipeline .getRootTransformIdsList() .stream() .filter(id -> !viewTransforms.contains(id))
.collect(Collectors.toList()); Pipeline res = Pipeline.newBuilder() .setComponents(fusedComponents) .addAllRootTransformIds(rootTransformIds)
GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
RunnerApi.Components components = pipeline.getComponents(); Map<String, String> outputs = transform.getTransform().getOutputsMap();
.build(); Pipeline impulse = Pipeline.newBuilder() .addRootTransformIds(impulse1Transform.getUniqueName()) .addRootTransformIds(impulse2Transform.getUniqueName())
RunnerApi.Components components = pipeline.getComponents(); Map<String, String> outputs = transform.getTransform().getOutputsMap();
RunnerApi.Components components = pipeline.getComponents(); RunnerApi.PTransform transform = components.getTransformsOrThrow(id); Map<String, String> outputs = transform.getOutputsMap();
private static <K, V> void translateGroupByKey( PTransformNode transform, RunnerApi.Pipeline pipeline, BatchTranslationContext context) { RunnerApi.Components components = pipeline.getComponents(); String inputPCollectionId = Iterables.getOnlyElement(transform.getTransform().getInputsMap().values()); RunnerApi.WindowingStrategy windowingStrategyProto = pipeline .getComponents() .getWindowingStrategiesOrThrow( pipeline .getComponents() .getPcollectionsOrThrow(inputPCollectionId) .getWindowingStrategyId()); RehydratedComponents.forComponents(pipeline.getComponents()); inputCoder = (WindowedValueCoder) WireCoders.instantiateRunnerWireCoder(inputCollection, pipeline.getComponents()); } catch (IOException e) { throw new RuntimeException(e);
FusedPipeline fused = GreedyPipelineFuser.fuse( Pipeline.newBuilder() .addRootTransformIds("impulse") .addRootTransformIds("compositeMultiLang")
.build()); Pipeline pipeline = Pipeline.newBuilder() .addRootTransformIds(inputTransform.getId()) .addRootTransformIds(groupByKeyOnly.getId()) new GroupByKeyOnlyEvaluatorFactory(graph, pipeline.getComponents(), bundleFactory) .forApplication(groupByKeyOnly, inputBundle);
private static <K, V> void translateGroupByKey( PTransformNode transform, RunnerApi.Pipeline pipeline, BatchTranslationContext context) { RunnerApi.Components components = pipeline.getComponents(); String inputPCollectionId = Iterables.getOnlyElement(transform.getTransform().getInputsMap().values()); RunnerApi.WindowingStrategy windowingStrategyProto = pipeline .getComponents() .getWindowingStrategiesOrThrow( pipeline .getComponents() .getPcollectionsOrThrow(inputPCollectionId) .getWindowingStrategyId()); RehydratedComponents.forComponents(pipeline.getComponents()); inputCoder = (WindowedValueCoder) WireCoders.instantiateRunnerWireCoder(inputCollection, pipeline.getComponents()); } catch (IOException e) { throw new RuntimeException(e);
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
@Test public void replacesOnlyMatching() { RunnerApi.Pipeline p = Pipeline.newBuilder() .addAllRootTransformIds(ImmutableList.of("first", "second")) .setComponents( ProtoOverrides.updateTransform( "beam:second", p, new TestReplacer(secondReplacement, extraComponents)); PTransform updatedSecond = updated.getComponents().getTransformsOrThrow("second"); updated.getComponents().getWindowingStrategiesOrThrow("new_ws"), equalTo(introducedWS)); assertThat( updated.getComponents().getTransformsOrThrow("second_sub"), equalTo(PTransform.getDefaultInstance())); updated.getComponents().getPcollectionsOrThrow("intermediatePc").getUniqueName(), equalTo("intermediate_replacement")); updated.getComponents().getTransformsOrThrow("first"), equalTo(p.getComponents().getTransformsOrThrow("first"))); assertThat( updated.getComponents().getCodersOrThrow("coder"), equalTo(p.getComponents().getCodersOrThrow("coder"))); assertThat(updated.getRootTransformIdsList(), equalTo(p.getRootTransformIdsList()));
RunnerApi.Pipeline.newBuilder() .addRootTransformIds(source.getId()) .addRootTransformIds(flatten.getId())
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());
.build(); FusedPipeline fused = GreedyPipelineFuser.fuse(Pipeline.newBuilder().setComponents(components).build());