private FsWriterMetrics buildWriterMetrics(String fileName, String partitionKey, int writerId, int numRecords) { return buildWriterMetrics(fileName, partitionKey, writerId, 0, numRecords); }
FsWriterMetrics metrics1 = buildWriterMetrics("newfile.json", null, 0, 90); wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics1.toJson()); wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md);
@Test public void testNoOutputWhenDisabledWithPartitions() throws IOException { File publishPath = Files.createTempDir(); State s = buildDefaultState(1); s.removeProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_DIR); s.removeProp(ConfigurationKeys.DATA_PUBLISHER_METADATA_OUTPUT_FILE); s.setProp(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, publishPath.getAbsolutePath()); WorkUnitState wuState = new WorkUnitState(); addStateToWorkunit(s, wuState); wuState.setProp(ConfigurationKeys.WRITER_METADATA_KEY, "abcdefg"); FsWriterMetrics metrics1 = buildWriterMetrics("foo1.json", "1-2-3-4", 0, 10); FsWriterMetrics metrics2 = buildWriterMetrics("foo1.json", "5-6-7-8",10, 20); wuState.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "1-2-3-4"); wuState.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics1.toJson()); wuState.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY + "_0", "1-2-3-4"); wuState.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + " _0", metrics2.toJson()); wuState.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY + "_1", "5-6-7-8"); wuState.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + " _1", metrics2.toJson()); BaseDataPublisher publisher = new BaseDataPublisher(s); publisher.publishMetadata(Collections.singletonList(wuState)); String[] filesInPublishDir = publishPath.list(); Assert.assertEquals(0, filesInPublishDir.length, "Expected 0 files to be output to publish path"); }
FsWriterMetrics metrics1 = buildWriterMetrics("foo1.json", "1-2-3-4", 0, 10); FsWriterMetrics metrics2 = buildWriterMetrics("foo1.json", "5-6-7-8",10, 20); wuState1.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY + ".0", "1-2-3-4"); wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY + ".0", metrics1.toJson()); FsWriterMetrics metrics3 = buildWriterMetrics("foo3.json", "1-2-3-4", 1, 1, 30); wuState2.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY + ".1", "1-2-3-4"); wuState2.setProp(ConfigurationKeys.WRITER_METADATA_KEY + ".1", md); FsWriterMetrics metrics4 = buildWriterMetrics("foo4.json", "5-6-7-8", 2, 55); wuState3.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY + ".0", "5-6-7-8"); wuState3.setProp(ConfigurationKeys.WRITER_METADATA_KEY + ".0", md);
FsWriterMetrics metrics1 = buildWriterMetrics("foo1.json", null, 0, 10); wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics1.toJson()); wuState1.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); FsWriterMetrics metrics3 = buildWriterMetrics("foo3.json", null, 1, 30); wuState2.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); wuState2.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics3.toJson()); FsWriterMetrics metrics4 = buildWriterMetrics("foo4.json", null, 2, 55); wuState3.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); wuState3.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics4.toJson());
FsWriterMetrics metrics1 = buildWriterMetrics("foo1.json", "1-2-3-4", 0, 10); FsWriterMetrics metrics2 = buildWriterMetrics("foo1.json", "5-6-7-8",10, 20); wuState1.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "1-2-3-4"); wuState1.setProp(FsDataWriter.FS_WRITER_METRICS_KEY, metrics1.toJson()); FsWriterMetrics metrics3 = buildWriterMetrics("foo3.json", "1-2-3-4", 1, 30); wuState2.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "1-2-3-4"); wuState2.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md); FsWriterMetrics metrics4 = buildWriterMetrics("foo4.json", "5-6-7-8", 2, 55); wuState3.setProp(ConfigurationKeys.WRITER_PARTITION_PATH_KEY, "5-6-7-8"); wuState3.setProp(ConfigurationKeys.WRITER_METADATA_KEY, md);