/** Converts the provided Json{@link String} into {@link PipelineOptions}. */ public static PipelineOptions fromJson(String optionsJson) { try { Map<String, Object> probingOptionsMap = MAPPER.readValue(optionsJson, new TypeReference<Map<String, Object>>() {}); if (probingOptionsMap.containsKey("options")) { //Legacy options. return MAPPER.readValue(optionsJson, PipelineOptions.class); } else { // Fn Options with namespace and version. Struct.Builder builder = Struct.newBuilder(); JsonFormat.parser().merge(optionsJson, builder); return fromProto(builder.build()); } } catch (IOException e) { throw new RuntimeException("Failed to read PipelineOptions from JSON", e); } }
@Test public void emptyStructDeserializes() throws Exception { Struct serialized = Struct.getDefaultInstance(); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized, notNullValue()); }
@Test public void testToFromProto() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); PipelineOptions deserializedStruct = PipelineOptionsTranslation.fromProto(originalStruct); Struct reserializedStruct = PipelineOptionsTranslation.toProto(deserializedStruct); assertThat(reserializedStruct.getFieldsMap(), equalTo(originalStruct.getFieldsMap())); }
@Test public void testPrepareJob() throws Exception { PrepareJobResponse response = stub.prepare( PrepareJobRequest.newBuilder() .setPipelineOptions(Struct.getDefaultInstance()) .setPipeline(Pipeline.getDefaultInstance()) .setJobName("myJobName") .build()); ApiServiceDescriptor stagingEndpoint = response.getArtifactStagingEndpoint(); ArtifactServiceStager stager = ArtifactServiceStager.overChannel( InProcessChannelBuilder.forName(stagingEndpoint.getUrl()).build()); String stagingSessionToken = "token"; File foo = writeTempFile("foo", "foo, bar, baz".getBytes(UTF_8)); File bar = writeTempFile("spam", "spam, ham, eggs".getBytes(UTF_8)); stager.stage( stagingSessionToken, ImmutableList.of(StagedFile.of(foo, foo.getName()), StagedFile.of(bar, bar.getName()))); List<byte[]> tempDirFiles = readFlattenedFiles(runnerTemp.getRoot()); assertThat( tempDirFiles, hasItems( arrayEquals(Files.readAllBytes(foo.toPath())), arrayEquals(Files.readAllBytes(bar.toPath())))); // TODO: 'run' the job with some sort of noop backend, to verify state is cleaned up. }
@Test public void testToFromJson() throws Exception { options.getOptionsId(); Struct originalStruct = PipelineOptionsTranslation.toProto(options); String json = PipelineOptionsTranslation.toJson(options); String legacyJson = MAPPER.writeValueAsString(options); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(json)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); assertThat( PipelineOptionsTranslation.toProto(PipelineOptionsTranslation.fromJson(legacyJson)) .getFieldsMap(), equalTo(originalStruct.getFieldsMap())); } }
/** Converts the provided {@link PipelineOptions} to a {@link Struct}. */ public static Struct toProto(PipelineOptions options) { Struct.Builder builder = Struct.newBuilder(); try { // TODO: Officially define URNs for options and their scheme. TreeNode treeNode = MAPPER.valueToTree(options); TreeNode rootOptions = treeNode.get("options"); Iterator<String> optionsKeys = rootOptions.fieldNames(); Map<String, TreeNode> optionsUsingUrns = new HashMap<>(); while (optionsKeys.hasNext()) { String optionKey = optionsKeys.next(); TreeNode optionValue = rootOptions.get(optionKey); optionsUsingUrns.put( "beam:option:" + CaseFormat.LOWER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, optionKey) + ":v1", optionValue); } // The JSON format of a Protobuf Struct is the JSON object that is equivalent to that struct // (with values encoded in a standard json-codeable manner). See Beam PR 3719 for more. JsonFormat.parser().merge(MAPPER.writeValueAsString(optionsUsingUrns), builder); return builder.build(); } catch (IOException e) { throw new RuntimeException("Failed to convert PipelineOptions to Protocol", e); } }
@Test public void structWithNullOptionsDeserializes() throws Exception { Struct serialized = Struct.newBuilder() .putFields( "beam:option:option_key:v1", Value.newBuilder().setNullValue(NullValue.NULL_VALUE).build()) .build(); PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized); assertThat(deserialized, notNullValue()); } }