@Test public void testToFromProto() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); PipelineOptions deserializedStruct = PipelineOptionsTranslation.fromProto(originalStruct); Struct reserializedStruct = PipelineOptionsTranslation.toProto(deserializedStruct); assertThat(reserializedStruct.getFieldsMap(), equalTo(originalStruct.getFieldsMap())); }
@Test public void testToFromJson() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); String json = PipelineOptionsTranslation.toJson(options); String legacyJson = MAPPER.writeValueAsString(options); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(json)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(legacyJson)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); } }
/** Converts the provided {@link PipelineOptions} into Json{@link String}. */ public static String toJson(PipelineOptions options) { try { return JsonFormat.printer().print(toProto(options)); } catch (InvalidProtocolBufferException e) { throw new RuntimeException("Failed to convert PipelineOptions to JSON", e); } } }
/** Converts the provided Json{@link String} into {@link PipelineOptions}. */ public static PipelineOptions fromJson(String optionsJson) { try { Map<String, Object> probingOptionsMap = MAPPER.readValue(optionsJson, new TypeReference<Map<String, Object>>() {}); if (probingOptionsMap.containsKey("options")) { //Legacy options. return MAPPER.readValue(optionsJson, PipelineOptions.class); } else { // Fn Options with namespace and version. Struct.Builder builder = Struct.newBuilder(); JsonFormat.parser().merge(optionsJson, builder); return fromProto(builder.build()); } } catch (IOException e) { throw new RuntimeException("Failed to read PipelineOptions from JSON", e); } }
private static JobInfo constructJobInfo(String jobId, long parallelism) { PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class); portableOptions.setSdkWorkerParallelism(parallelism); Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions); return JobInfo.create(jobId, "job-name", "retrieval-token", pipelineOptions); }
@Override public FlinkExecutableStageContext get(JobInfo jobInfo) { JobFactoryState state = jobFactories.computeIfAbsent( jobInfo.jobId(), k -> { PortablePipelineOptions portableOptions = PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions()) .as(PortablePipelineOptions.class); return new JobFactoryState( MoreObjects.firstNonNull(portableOptions.getSdkWorkerParallelism(), 1L) .intValue()); }); return state.getFactory().get(jobInfo); } }
@Test public void defaultsRestored() throws Exception { Struct serialized = PipelineOptionsTranslation.toProto(PipelineOptionsFactory.as(TestDefaultOptions.class)); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized.as(TestDefaultOptions.class).getDefault(), equalTo(19)); }
private static JobInfo constructJobInfo(String jobId, long parallelism) { PortablePipelineOptions portableOptions = PipelineOptionsFactory.as(PortablePipelineOptions.class); portableOptions.setSdkWorkerParallelism(parallelism); Struct pipelineOptions = PipelineOptionsTranslation.toProto(portableOptions); return JobInfo.create(jobId, "job-name", "retrieval-token", pipelineOptions); }
@Override public FlinkExecutableStageContext get(JobInfo jobInfo) { JobFactoryState state = jobFactories.computeIfAbsent( jobInfo.jobId(), k -> { PortablePipelineOptions portableOptions = PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions()) .as(PortablePipelineOptions.class); return new JobFactoryState( MoreObjects.firstNonNull(portableOptions.getSdkWorkerParallelism(), 1L) .intValue()); }); return state.getFactory().get(jobInfo); } }
@Test public void ignoredSettingsNotSerialized() throws Exception { TestUnserializableOptions opts = PipelineOptionsFactory.as(TestUnserializableOptions.class); opts.setUnserializable(new Object()); Struct serialized = PipelineOptionsTranslation.toProto(opts); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat( deserialized.as(TestUnserializableOptions.class).getUnserializable(), is(nullValue())); }
private <T extends FlinkPortablePipelineTranslator.TranslationContext> PipelineResult runPipelineWithTranslator(FlinkPortablePipelineTranslator<T> translator) throws Exception { LOG.info("Translating pipeline to Flink program."); // Don't let the fuser fuse any subcomponents of native transforms. // TODO(BEAM-6327): Remove the need for this. RunnerApi.Pipeline trimmedPipeline = makeKnownUrnsPrimitives( pipeline, Sets.difference( translator.knownUrns(), ImmutableSet.of(PTransformTranslation.ASSIGN_WINDOWS_TRANSFORM_URN))); // Fused pipeline proto. RunnerApi.Pipeline fusedPipeline = GreedyPipelineFuser.fuse(trimmedPipeline).toPipeline(); JobInfo jobInfo = JobInfo.create( id, pipelineOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(pipelineOptions)); FlinkPortablePipelineTranslator.Executor executor = translator.translate( translator.createTranslationContext(jobInfo, pipelineOptions, confDir, filesToStage), fusedPipeline); final JobExecutionResult result = executor.execute(pipelineOptions.getJobName()); return FlinkRunner.createPipelineResult(result, pipelineOptions); }
PipelineOptionsTranslation.fromProto(options).as(FlinkPipelineOptions.class);
@Test public void customSettingsRetained() throws Exception { TestOptions options = PipelineOptionsFactory.as(TestOptions.class); options.setExample(23); Struct serialized = PipelineOptionsTranslation.toProto(options); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized.as(TestOptions.class).getExample(), equalTo(23)); }
.setJobName(options.getJobName()) .setPipeline(PipelineTranslation.toProto(pipeline)) .setPipelineOptions(PipelineOptionsTranslation.toProto(options)) .build();
PipelineOptionsTranslation.fromProto(options).as(FlinkPipelineOptions.class);
pipelineOptions.getJobName(), retrievalToken, PipelineOptionsTranslation.toProto(pipelineOptions)); final JobExecutionResult result;
@SuppressWarnings("FutureReturnValueIgnored") private void scheduleRelease(JobInfo jobInfo) { WrappedContext wrapper = getCache().get(jobInfo.jobId()); Preconditions.checkState( wrapper != null, "Releasing context for unknown job: " + jobInfo.jobId()); PipelineOptions pipelineOptions = PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions()); int environmentCacheTTLMillis = pipelineOptions.as(PortablePipelineOptions.class).getEnvironmentCacheMillis(); if (environmentCacheTTLMillis > 0) { // Do immediate cleanup if this class is not loaded on Flink parent classloader. if (this.getClass().getClassLoader() != ExecutionEnvironment.class.getClassLoader()) { LOG.warn( "{} is not loaded on parent Flink classloader. " + "Falling back to synchronous environment release for job {}.", this.getClass(), jobInfo.jobId()); release(wrapper); } else { // Schedule task to clean the container later. // Ensure that this class is loaded in the parent Flink classloader. getExecutor() .schedule(() -> release(wrapper), environmentCacheTTLMillis, TimeUnit.MILLISECONDS); } } else { // Do not release this asynchronously, as the releasing could fail due to the classloader not // being available anymore after the tasks have been removed from the execution engine. release(wrapper); } }
@SuppressWarnings("FutureReturnValueIgnored") private void scheduleRelease(JobInfo jobInfo) { WrappedContext wrapper = getCache().get(jobInfo.jobId()); Preconditions.checkState( wrapper != null, "Releasing context for unknown job: " + jobInfo.jobId()); PipelineOptions pipelineOptions = PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions()); int environmentCacheTTLMillis = pipelineOptions.as(PortablePipelineOptions.class).getEnvironmentCacheMillis(); if (environmentCacheTTLMillis > 0) { // Do immediate cleanup if this class is not loaded on Flink parent classloader. if (this.getClass().getClassLoader() != ExecutionEnvironment.class.getClassLoader()) { LOG.warn( "{} is not loaded on parent Flink classloader. " + "Falling back to synchronous environment release for job {}.", this.getClass(), jobInfo.jobId()); release(wrapper); } else { // Schedule task to clean the container later. // Ensure that this class is loaded in the parent Flink classloader. getExecutor() .schedule(() -> release(wrapper), environmentCacheTTLMillis, TimeUnit.MILLISECONDS); } } else { // Do not release this asynchronously, as the releasing could fail due to the classloader not // being available anymore after the tasks have been removed from the execution engine. release(wrapper); } }
@Test public void emptyStructDeserializes() throws Exception { Struct serialized = Struct.getDefaultInstance(); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized, notNullValue()); }
private static FlinkDefaultExecutableStageContext create(JobInfo jobInfo) { JobBundleFactory jobBundleFactory = DefaultJobBundleFactory.create( jobInfo, ImmutableMap.of( BeamUrns.getUrn(StandardEnvironments.Environments.DOCKER), new DockerEnvironmentFactory.Provider( PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions())), BeamUrns.getUrn(StandardEnvironments.Environments.PROCESS), new ProcessEnvironmentFactory.Provider(), BeamUrns.getUrn(StandardEnvironments.Environments.EXTERNAL), new ExternalEnvironmentFactory.Provider(), Environments.ENVIRONMENT_EMBEDDED, // Non Public urn for testing. new EmbeddedEnvironmentFactory.Provider( PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions())))); return new FlinkDefaultExecutableStageContext(jobBundleFactory); }