/** Converts the provided {@link PipelineOptions} into Json{@link String}. */ public static String toJson(PipelineOptions options) { try { return JsonFormat.printer().print(toProto(options)); } catch (InvalidProtocolBufferException e) { throw new RuntimeException("Failed to convert PipelineOptions to JSON", e); } } }
@Test public void testToFromProto() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); PipelineOptions deserializedStruct = PipelineOptionsTranslation.fromProto(originalStruct); Struct reserializedStruct = PipelineOptionsTranslation.toProto(deserializedStruct); assertThat(reserializedStruct.getFieldsMap(), equalTo(originalStruct.getFieldsMap())); }
private static JobInfo constructJobInfo(String jobId, long parallelism) { PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class); portableOptions.setSdkWorkerParallelism(parallelism); Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions); return JobInfo.create(jobId, "job-name", "retrieval-token", pipelineOptions); }
private static JobInfo constructJobInfo(String jobId, long parallelism) { PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class); portableOptions.setSdkWorkerParallelism(parallelism); Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions); return JobInfo.create(jobId, "job-name", "retrieval-token", pipelineOptions); }
@Test public void testToFromJson() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); String json = PipelineOptionsTranslation.toJson(options); String legacyJson = MAPPER.writeValueAsString(options); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(json)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(legacyJson)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); } }
@Test public void ignoredSettingsNotSerialized() throws Exception { TestUnserializableOptions opts = PipelineOptionsFactory.as(TestUnserializableOptions.class); opts.setUnserializable(new Object()); Struct serialized = PipelineOptionsTranslation.toProto(opts); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat( deserialized.as(TestUnserializableOptions.class).getUnserializable(), is(nullValue())); }
@Test public void defaultsRestored() throws Exception { Struct serialized = PipelineOptionsTranslation.toProto(PipelineOptionsFactory.as(TestDefaultOptions.class)); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized.as(TestDefaultOptions.class).getDefault(), equalTo(19)); }
@Test public void customSettingsRetained() throws Exception { TestOptions options = PipelineOptionsFactory.as(TestOptions.class); options.setExample(23); Struct serialized = PipelineOptionsTranslation.toProto(options); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized.as(TestOptions.class).getExample(), equalTo(23)); }
private <T extends FlinkPortablePipelineTranslator.TranslationContext> PipelineResult runPipelineWithTranslator(FlinkPortablePipelineTranslator<T> translator) throws Exception { LOG.info("Translating pipeline to Flink program."); // Don't let the fuser fuse any subcomponents of native transforms. // TODO(BEAM-6327): Remove the need for this. RunnerApi.Pipeline trimmedPipeline = makeKnownUrnsPrimitives( pipeline, Sets.difference( translator.knownUrns(), ImmutableSet.of(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN))); // Fused pipeline proto. RunnerApi.Pipeline fusedPipeline = GreedyPipelineFuser.fuse(trimmedPipeline).toPipeline(); JobInfo jobInfo = JobInfo.create( id, pipelineOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(pipelineOptions)); FlinkPortablePipelineTranslator.Executor executor = translator.translate( translator.createTranslationContext(jobInfo, pipelineOptions, confDir, filesToStage), fusedPipeline); final JobExecutionResult result = executor.execute(pipelineOptions.getJobName()); return FlinkRunner.createPipelineResult(result, pipelineOptions); }
.setJobName(options.getJobName()) .setPipeline(PipelineTranslation.toProto(pipeline)) .setPipelineOptions(PipelineOptionsTranslation.toProto(options)) .build();
pipelineOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(pipelineOptions)); final JobExecutionResult result;