this.reader = new ParquetFileReader( configuration, footer.getFileMetaData(), path, blocks, requestedSchema.getColumns());
this.reader = new ParquetFileReader( config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); for (BlockMetaData block : blocks) {
this.reader = new ParquetFileReader( config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
this.reader = new ParquetFileReader( config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
configuration.get(ParquetReadSupport$.MODULE$.SPARK_ROW_REQUESTED_SCHEMA()); this.sparkSchema = StructType$.MODULE$.fromString(sparkRequestedSchemaString); this.reader = new ParquetFileReader( configuration, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); for (BlockMetaData block : blocks) {
configuration.get(ParquetReadSupport$.MODULE$.SPARK_ROW_REQUESTED_SCHEMA()); this.sparkSchema = StructType$.MODULE$.fromString(sparkRequestedSchemaString); this.reader = new ParquetFileReader( configuration, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
configuration.get(ParquetReadSupport$.MODULE$.SPARK_ROW_REQUESTED_SCHEMA()); this.sparkSchema = StructType$.MODULE$.fromString(sparkRequestedSchemaString); this.reader = new ParquetFileReader( configuration, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
/** * Open a {@link InputFile file} with {@link ParquetReadOptions options}. * * @param file an input file * @param options parquet read options * @return an open ParquetFileReader * @throws IOException if there is an error while opening the file */ public static ParquetFileReader open(InputFile file, ParquetReadOptions options) throws IOException { return new ParquetFileReader(file, options); }
/** * Open a {@link InputFile file} with {@link ParquetReadOptions options}. * * @param file an input file * @param options parquet read options * @return an open ParquetFileReader * @throws IOException if there is an error while opening the file */ public static ParquetFileReader open(InputFile file, ParquetReadOptions options) throws IOException { return new ParquetFileReader(file, options); }
/** * @param conf a configuration * @param file a file path to open * @param footer a footer for the file if already loaded * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0 */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file, ParquetMetadata footer) throws IOException { return new ParquetFileReader(conf, file, footer); }
/** * @param conf a configuration * @param file a file path to open * @param footer a footer for the file if already loaded * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0 */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file, ParquetMetadata footer) throws IOException { return new ParquetFileReader(conf, file, footer); }
/** * Open a {@link InputFile file}. * * @param file an input file * @return an open ParquetFileReader * @throws IOException if there is an error while opening the file */ public static ParquetFileReader open(InputFile file) throws IOException { return new ParquetFileReader(file, ParquetReadOptions.builder().build()); }
/** * Open a {@link InputFile file}. * * @param file an input file * @return an open ParquetFileReader * @throws IOException if there is an error while opening the file */ public static ParquetFileReader open(InputFile file) throws IOException { return new ParquetFileReader(file, ParquetReadOptions.builder().build()); }
/** * @param conf a configuration * @param file a file path to open * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0; use {@link #open(InputFile)} */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file) throws IOException { return new ParquetFileReader(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).build()); }
/** * @param conf a configuration * @param file a file path to open * @return a parquet file reader * @throws IOException if there is an error while opening the file * @deprecated will be removed in 2.0.0; use {@link #open(InputFile)} */ @Deprecated public static ParquetFileReader open(Configuration conf, Path file) throws IOException { return new ParquetFileReader(HadoopInputFile.fromPath(file, conf), HadoopReadOptions.builder(conf).build()); }
this.reader = new ParquetFileReader( config, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); for (BlockMetaData block : blocks) {
this.reader = new ParquetFileReader( configuration, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns());
public void initialize(FileMetaData parquetFileMetadata, Path file, List<BlockMetaData> blocks, Configuration configuration) throws IOException { // initialize a ReadContext for this file Map<String, String> fileMetadata = parquetFileMetadata.getKeyValueMetaData(); ReadSupport.ReadContext readContext = readSupport.init(new InitContext( configuration, toSetMultiMap(fileMetadata), fileSchema)); this.columnIOFactory = new ColumnIOFactory(parquetFileMetadata.getCreatedBy()); this.requestedSchema = readContext.getRequestedSchema(); this.fileSchema = parquetFileMetadata.getSchema(); this.file = file; this.columnCount = requestedSchema.getPaths().size(); this.recordConverter = readSupport.prepareForRead( configuration, fileMetadata, fileSchema, readContext); this.strictTypeChecking = configuration.getBoolean(STRICT_TYPE_CHECKING, true); List<ColumnDescriptor> columns = requestedSchema.getColumns(); reader = new ParquetFileReader(configuration, parquetFileMetadata, file, blocks, columns); for (BlockMetaData block : blocks) { total += block.getRowCount(); } this.unmaterializableRecordCounter = new UnmaterializableRecordCounter(configuration, total); LOG.info("RecordReader initialized will read a total of " + total + " records."); }
public void initialize(FileMetaData parquetFileMetadata, Path file, List<BlockMetaData> blocks, Configuration configuration) throws IOException { // initialize a ReadContext for this file Map<String, String> fileMetadata = parquetFileMetadata.getKeyValueMetaData(); ReadSupport.ReadContext readContext = readSupport.init(new InitContext( configuration, toSetMultiMap(fileMetadata), fileSchema)); this.columnIOFactory = new ColumnIOFactory(parquetFileMetadata.getCreatedBy()); this.requestedSchema = readContext.getRequestedSchema(); this.fileSchema = parquetFileMetadata.getSchema(); this.file = file; this.columnCount = requestedSchema.getPaths().size(); this.recordConverter = readSupport.prepareForRead( configuration, fileMetadata, fileSchema, readContext); this.strictTypeChecking = configuration.getBoolean(STRICT_TYPE_CHECKING, true); List<ColumnDescriptor> columns = requestedSchema.getColumns(); reader = new ParquetFileReader(configuration, parquetFileMetadata, file, blocks, columns); for (BlockMetaData block : blocks) { total += block.getRowCount(); } this.unmaterializableRecordCounter = new UnmaterializableRecordCounter(configuration, total); LOG.info("RecordReader initialized will read a total of " + total + " records."); }
configuration.get(ParquetReadSupport$.MODULE$.SPARK_ROW_REQUESTED_SCHEMA()); this.sparkSchema = StructType$.MODULE$.fromString(sparkRequestedSchemaString); this.reader = new ParquetFileReader( configuration, footer.getFileMetaData(), file, blocks, requestedSchema.getColumns()); for (BlockMetaData block : blocks) {