@Deprecated public static List<Footer> readAllFootersInParallel(final Configuration configuration, List<FileStatus> partFiles) throws IOException { return readAllFootersInParallel(configuration, partFiles, false); }
/** * Read the footers of all the files under that path (recursively) * not using summary files. * rowGroups are not skipped * @param configuration the configuration to access the FS * @param fileStatus the root dir * @return all the footers * @throws IOException */ public static List<Footer> readAllFootersInParallel(Configuration configuration, FileStatus fileStatus) throws IOException { List<FileStatus> statuses = listFiles(configuration, fileStatus); return readAllFootersInParallel(configuration, statuses, false); }
public static void writeMetaDataFile(Configuration configuration, Path outputPath) { if (configuration.getBoolean(ParquetOutputFormat.ENABLE_JOB_SUMMARY, true)) { try { final FileSystem fileSystem = outputPath.getFileSystem(configuration); FileStatus outputStatus = fileSystem.getFileStatus(outputPath); List<Footer> footers = ParquetFileReader.readAllFootersInParallel(configuration, outputStatus); try { ParquetFileWriter.writeMetadataFile(configuration, outputPath, footers); } catch (Exception e) { LOG.warn("could not write summary file for " + outputPath, e); final Path metadataPath = new Path(outputPath, ParquetFileWriter.PARQUET_METADATA_FILE); if (fileSystem.exists(metadataPath)) { fileSystem.delete(metadataPath, true); } } } catch (Exception e) { LOG.warn("could not write summary file for " + outputPath, e); } } }
result.addAll(readAllFootersInParallel(configuration, toRead, skipRowGroups));
List<Footer> footers = ParquetFileReader.readAllFootersInParallel(getConf(), files, false); for (Footer footer : footers) { Path path = footer.getFile();