this.id, new PartitionIdentifier(this.partitionKey, this.branchId), ImmutableSet.of(new FsWriterMetrics.FileInfo(this.outputFile.getName(), recordsWritten())) ); this.properties.setProp(FS_WRITER_METRICS_KEY, metrics.toJson());
private void checkMetadata(File file, int expectedNumFiles, int expectedNumRecords, FsWriterMetrics.FileInfo... expectedFileInfo) throws IOException { Assert.assertTrue(file.exists(), "Expected file " + file.getAbsolutePath() + " to exist"); String contents = IOUtils.toString(new FileInputStream(file), StandardCharsets.UTF_8); GlobalMetadata metadata = GlobalMetadata.fromJson(contents); Assert.assertEquals(metadata.getNumFiles(), expectedNumFiles, "# of files do not match"); Assert.assertEquals(metadata.getNumRecords(), expectedNumRecords, "# of records do not match"); for (FsWriterMetrics.FileInfo fileInfo : expectedFileInfo) { long recordsInMetadata = ((Number) metadata.getFileMetadata(fileInfo.getFileName(), GlobalMetadata.NUM_RECORDS_KEY)).longValue(); Assert.assertEquals(recordsInMetadata, fileInfo.getNumRecords(), "# of records in file-level metadata do not match"); } }
@Override public void close() throws IOException { this.closer.close(); if (this.shouldIncludeRecordCountInFileName) { String filePathWithRecordCount = addRecordCountToFileName(); this.properties.appendToSetProp(this.allOutputFilesPropName, filePathWithRecordCount); } else { this.properties.appendToSetProp(this.allOutputFilesPropName, getOutputFilePath()); } FsWriterMetrics metrics = new FsWriterMetrics( this.id, new PartitionIdentifier(this.partitionKey, this.branchId), ImmutableSet.of(new FsWriterMetrics.FileInfo(this.outputFile.getName(), recordsWritten())) ); this.properties.setProp(FS_WRITER_METRICS_KEY, metrics.toJson()); }
@Override public void update(FsWriterMetrics metrics) { long numRecords = mergedMetadata.getNumRecords(); int numFiles = mergedMetadata.getNumFiles(); for (FsWriterMetrics.FileInfo fileInfo: metrics.getFileInfos()) { numRecords += fileInfo.getNumRecords(); numFiles += 1; mergedMetadata.setFileMetadata(fileInfo.getFileName(), GlobalMetadata.NUM_RECORDS_KEY, Long.valueOf(fileInfo.getNumRecords())); } mergedMetadata.setNumRecords(numRecords); mergedMetadata.setNumOutputFiles(numFiles); }
new FsWriterMetrics.FileInfo("foo1.json", 10)); checkMetadata(new File(branchPaths.get(0)[1], "metadata.json.0"), 2, 75, new FsWriterMetrics.FileInfo("foo1.json", 20), new FsWriterMetrics.FileInfo("foo4.json", 55)); checkMetadata(new File(branchPaths.get(1)[0], "metadata.json.1"), 1, 30, new FsWriterMetrics.FileInfo("foo3.json", 30)); } finally { Arrays.stream(publishPaths).forEach(dir -> {
new FsWriterMetrics.FileInfo("foo3.json", 30), new FsWriterMetrics.FileInfo("foo1.json", 10)); checkMetadata(new File(part2, "metadata.json"), 2, 75, new FsWriterMetrics.FileInfo("foo1.json", 20), new FsWriterMetrics.FileInfo("foo4.json", 55)); } finally { FileUtils.deleteDirectory(publishPath);
new FsWriterMetrics.FileInfo("foo3.json", 30), new FsWriterMetrics.FileInfo("foo1.json", 10), new FsWriterMetrics.FileInfo("foo4.json", 55)); } finally { FileUtils.deleteDirectory(publishPath);
new FsWriterMetrics.FileInfo("foo3.json", 30), new FsWriterMetrics.FileInfo("foo1.json", 10), new FsWriterMetrics.FileInfo("foo4.json", 55), new FsWriterMetrics.FileInfo("newfile.json", 90)); } finally { FileUtils.deleteDirectory(publishPath);
@Test public void testSerialization() throws IOException { final String WRITER_ID = "foobar123"; final PartitionIdentifier PARTITION_KEY = new PartitionIdentifier("_partitionInfo", 3); final Set<FsWriterMetrics.FileInfo> FILE_INFOS = ImmutableSet.of( new FsWriterMetrics.FileInfo("file1", 1234), new FsWriterMetrics.FileInfo("file2", 4321) ); String metricsJson = new FsWriterMetrics(WRITER_ID, PARTITION_KEY, FILE_INFOS).toJson(); FsWriterMetrics parsedMetrics = FsWriterMetrics.fromJson(metricsJson); Assert.assertEquals(parsedMetrics.writerId, WRITER_ID); Assert.assertEquals(parsedMetrics.partitionInfo, PARTITION_KEY); Assert.assertEquals(parsedMetrics.fileInfos, FILE_INFOS); } }
@Override public void update(FsWriterMetrics metrics) { long numRecords = mergedMetadata.getNumRecords(); int numFiles = mergedMetadata.getNumFiles(); for (FsWriterMetrics.FileInfo fileInfo: metrics.getFileInfos()) { numRecords += fileInfo.getNumRecords(); numFiles += 1; mergedMetadata.setFileMetadata(fileInfo.getFileName(), GlobalMetadata.NUM_RECORDS_KEY, Long.valueOf(fileInfo.getNumRecords())); } mergedMetadata.setNumRecords(numRecords); mergedMetadata.setNumOutputFiles(numFiles); }
private FsWriterMetrics buildWriterMetrics(String fileName, String partitionKey, int writerId, int branchId, int numRecords) { return new FsWriterMetrics( String.format("writer%d", writerId), new PartitionIdentifier(partitionKey, branchId), ImmutableList.of(new FsWriterMetrics.FileInfo(fileName, numRecords)) ); }