@Test public void testNoOutputWhenDisabled() throws IOException { State s = buildDefaultState(1); WorkUnitState wuState = new WorkUnitState(); addStateToWorkunit(s, wuState); wuState.setProp(ConfigurationKeys.WRITER_METADATA_KEY, "abcdefg"); BaseDataPublisher publisher = new BaseDataPublisher(s); publisher.publishMetadata(Collections.singletonList(wuState)); File mdFile = openMetadataFile(s, 1, 0); Assert.assertFalse(mdFile.exists(), "Internal metadata from writer should not be written out if no merger is set in config"); }
/** * Test that DATA_PUBLISHER_METADATA_STR functionality works across multiple branches. */ @Test public void testMetadataStrMultipleWorkUnitsAndBranches() throws IOException { final int numBranches = 3; State s = buildDefaultState(numBranches); List<WorkUnitState> workUnits = new ArrayList<>(); for (int i = 0; i < numBranches; i++) { WorkUnitState wuState = new WorkUnitState(); wuState.setProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_STR, "foobar"); addStateToWorkunit(s, wuState); workUnits.add(wuState); } BaseDataPublisher publisher = new BaseDataPublisher(s); publisher.publishMetadata(workUnits); for (int branch = 0; branch < numBranches; branch++) { try (InputStream mdStream = new FileInputStream(openMetadataFile(s, numBranches, branch))) { String mdBytes = IOUtils.toString(mdStream, StandardCharsets.UTF_8); Assert.assertEquals(mdBytes, "foobar", "Expected to read back metadata from string"); } } }
/** * Test DATA_PUBLISHER_METADATA_STR: a user should be able to put an arbitrary metadata string in job configuration * and have that written out. */ @Test public void testMetadataStrOneBranch() throws IOException { State s = buildDefaultState(1); WorkUnitState wuState = new WorkUnitState(); wuState.setProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_STR, "foobar"); addStateToWorkunit(s, wuState); BaseDataPublisher publisher = new BaseDataPublisher(s); publisher.publishMetadata(wuState); try (InputStream mdStream = new FileInputStream(openMetadataFile(s, 1, 0))) { String mdBytes = IOUtils.toString(mdStream, StandardCharsets.UTF_8); Assert.assertEquals(mdBytes, "foobar", "Expected to read back metadata from string"); } }
wuState1.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "1-2-3-4"); wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); addStateToWorkunit(s, wuState1); addStateToWorkunit(s, wuState2);
/** * Test that an exception is properly thrown if we configure a merger that doesn't actually implement * MetadataMerger */ @Test(expectedExceptions = IllegalArgumentException.class) public void testBogusMetadataMerger() throws IOException { State s = buildDefaultState(1); s.setProp(ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_KEY, "true"); s.setProp(ConfigurationKeys.DATA_PUBLISH_WRITER_METADATA_MERGER_NAME_KEY, "java.lang.String"); s.setProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_STR, "foobar"); WorkUnitState wuState = new WorkUnitState(); addStateToWorkunit(s, wuState); BaseDataPublisher publisher = new BaseDataPublisher(s); publisher.publishMetadata(Collections.singletonList(wuState)); }
for (int workUnitId = 0; workUnitId < numWorkUnits; workUnitId++) { WorkUnitState wuState = new WorkUnitState(); addStateToWorkunit(s, wuState);
wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics1.toJson()); wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); addStateToWorkunit(s, wuState1);
wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + ".0_1", metrics2.toJson()); wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY + ".0", md); addStateToWorkunit(s, wuState1); wuState2.setProp(ConfigurationKeys.WRITER_METADATA_KEY + ".1", md); wuState2.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + ".1", metrics3.toJson()); addStateToWorkunit(s, wuState2); wuState3.setProp(ConfigurationKeys.WRITER_METADATA_KEY + ".0", md); wuState3.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + ".0", metrics4.toJson()); addStateToWorkunit(s, wuState3);
@Test public void testNoOutputWhenDisabledWithPartitions() throws IOException { File publishPath = Files.createTempDir(); State s = buildDefaultState(1); s.removeProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_DIR); s.removeProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_FILE); s.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, publishPath.getAbsolutePath()); WorkUnitState wuState = new WorkUnitState(); addStateToWorkunit(s, wuState); wuState.setProp(ConfigurationKeys.WRITER_METADATA_KEY, "abcdefg"); FsWriterMetrics metrics1 = buildWriterMetrics("foo1.json", "1-2-3-4", 0, 10); FsWriterMetrics metrics2 = buildWriterMetrics("foo1.json", "5-6-7-8",10, 20); wuState.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "1-2-3-4"); wuState.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics1.toJson()); wuState.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY + "_0", "1-2-3-4"); wuState.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + " _0", metrics2.toJson()); wuState.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY + "_1", "5-6-7-8"); wuState.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + " _1", metrics2.toJson()); BaseDataPublisher publisher = new BaseDataPublisher(s); publisher.publishMetadata(Collections.singletonList(wuState)); String[] filesInPublishDir = publishPath.list(); Assert.assertEquals(0, filesInPublishDir.length, "Expected 0 files to be output to publish path"); }
wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics1.toJson()); wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); addStateToWorkunit(s, wuState1); wuState2.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); wuState2.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics3.toJson()); addStateToWorkunit(s, wuState2); wuState3.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); wuState3.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics4.toJson()); addStateToWorkunit(s, wuState3);
wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + " _1", metrics2.toJson()); wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); addStateToWorkunit(s, wuState1); wuState2.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); wuState2.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics3.toJson()); addStateToWorkunit(s, wuState2); wuState3.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); wuState3.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics4.toJson()); addStateToWorkunit(s, wuState3);