.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", threeOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .setSpec(
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("pyImpulse.out", pc("pyImpulse.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("goImpulse.out", pc("goImpulse.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN))
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("mainImpulse.out", pc("mainImpulse.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("sideImpulse.out", pc("sideImpulse.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN)
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .build() .toByteString()) .build()) .build(); PTransform goRead = .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .build() .toByteString()) .build()) .build(); .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py"))
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("pyImpulse.out", pc("pyImpulse.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN)) .build()) .putPcollections("goImpulse.out", pc("goImpulse.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN))
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("two", twoOut.getUniqueName()) .setSpec(
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("red", redOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", sharedOut.getUniqueName()) .putOutputs("out", blueOut.getUniqueName())
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("rare")) .putInputs("otherEnvInput", "envRead.out") .putOutputs("output", "flatten.out") .setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN)) .build(); PTransform windowTransform = .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common"))
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.IMPULSE_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec(
.setSpec( SdkFunctionSpec.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn("beam:direct:keyedworkitem:v1"))) .addAllComponentCoderIds( ImmutableList.of( .setUniqueName(String.format("%s/RawGBK", spk.getUniqueName())) .putAllInputs(spk.getInputsMap()) .setSpec(FunctionSpec.newBuilder().setUrn(DirectGroupByKey.DIRECT_GBKO_URN)) .putOutputs("output", kwiCollectionId) .build(); .setSpec( FunctionSpec.newBuilder() .setUrn(SplittableRemoteStageEvaluatorFactory.FEED_SDF_URN)) .putOutputs("output", feedSDFCollectionId) .build(); .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.SPLITTABLE_PROCESS_ELEMENTS_URN) .setPayload(spk.getSpec().getPayload())) .putAllOutputs(spk.getOutputsMap()) .build();
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("go")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putOutputs("out", redOut.getUniqueName()) .build(); .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", oneOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("in", redOut.getUniqueName()) .putOutputs("out", twoOut.getUniqueName()) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .build()) .putInputs("one", oneOut.getUniqueName()) .putInputs("two", twoOut.getUniqueName())
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN)) .putInputs("input", "impulse.out") .putOutputs("output", "side_read.out")
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("common")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN)) .putInputs("input", "impulse.out") .putOutputs("output", "side_read.out") .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn( .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN) .setPayload( WindowIntoPayload.newBuilder() .setWindowFn(
"first", PTransform.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn("beam:first")) .build()) .putTransforms( "second", PTransform.newBuilder() .setSpec(FunctionSpec.newBuilder().setUrn("beam:second")) .build()) .putPcollections( .setSpec( FunctionSpec.newBuilder() .setUrn("beam:second:replacement") .setPayload( ByteString.copyFrom("foo-bar-baz".getBytes(StandardCharsets.UTF_8)))) .build();
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.GROUP_BY_KEY_TRANSFORM_URN)) .build()) .putPcollections("groupByKey.out", pc("groupByKey.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py"))
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("foo"))
.putOutputs("out", flattened.getId()) .setSpec( FunctionSpec.newBuilder().setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN)) .build());
.setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.FLATTEN_TRANSFORM_URN) .build()) .build()) .putPcollections("flatten.out", pc("flatten.out")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(SdkFunctionSpec.newBuilder().setEnvironmentId("py")) .setSpec( FunctionSpec.newBuilder() .setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN) .setPayload( ParDoPayload.newBuilder() .setDoFn(