/** * Initialize ORC file reader and batch record reader. * Please note that `initBatch` is needed to be called after this. */ @Override public void initialize( InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { FileSplit fileSplit = (FileSplit)inputSplit; Configuration conf = taskAttemptContext.getConfiguration(); Reader reader = OrcFile.createReader( fileSplit.getPath(), OrcFile.readerOptions(conf) .maxLength(OrcConf.MAX_FILE_LENGTH.getLong(conf)) .filesystem(fileSplit.getPath().getFileSystem(conf))); Reader.Options options = OrcInputFormat.buildOptions(conf, reader, fileSplit.getStart(), fileSplit.getLength()); recordReader = reader.rows(options); }
/** * Initialize ORC file reader and batch record reader. * Please note that `initBatch` is needed to be called after this. */ @Override public void initialize( InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException { FileSplit fileSplit = (FileSplit)inputSplit; Configuration conf = taskAttemptContext.getConfiguration(); Reader reader = OrcFile.createReader( fileSplit.getPath(), OrcFile.readerOptions(conf) .maxLength(OrcConf.MAX_FILE_LENGTH.getLong(conf)) .filesystem(fileSplit.getPath().getFileSystem(conf))); Reader.Options options = OrcInputFormat.buildOptions(conf, reader, fileSplit.getStart(), fileSplit.getLength()); recordReader = reader.rows(options); }
private static boolean needsCompaction(FileStatus bucket, FileSystem fs) throws IOException { //create reader, look at footer //no need to check side file since it can only be in a streaming ingest delta Reader orcReader = OrcFile.createReader(bucket.getPath(),OrcFile.readerOptions(fs.getConf()) .filesystem(fs)); AcidStats as = OrcAcidUtils.parseAcidStats(orcReader); if(as == null) { //should never happen since we are reading bucket_x written by acid write throw new IllegalStateException("AcidStats missing in " + bucket.getPath()); } return as.deletes > 0 || as.updates > 0; } private static String getCompactionCommand(Table t, Partition p) {
FileSystem fs = input.getFileSystem(conf); Reader reader = createReader(input, readerOptions(options.getConfiguration()).filesystem(fs));
OrcFile.readerOptions( conf ).filesystem( fs ) ); } catch ( IOException e ) { throw new IllegalArgumentException( "Unable to read data from file " + fileName, e );
OrcFile.readerOptions( conf ).filesystem( fs ) ); } catch ( IOException e ) { throw new RuntimeException( "Unable to read data from file " + fileName, e );
@Override public void initialize( InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException { FileSplit fileSplit = (FileSplit) inputSplit; Configuration conf = taskAttemptContext.getConfiguration(); Reader reader = OrcFile.createReader( fileSplit.getPath(), OrcFile.readerOptions(conf) .maxLength(OrcConf.MAX_FILE_LENGTH.getLong(conf)) .filesystem(fileSplit.getPath().getFileSystem(conf))); Reader.Options options = OrcInputFormat.buildOptions(conf, reader, fileSplit.getStart(), fileSplit.getLength()); recordReader = reader.rows(options); orcSchema = reader.getSchema(); requestedColumnIds = OrcUtils.requestedColumnIds(caseSensitive, fieldNames, schemaFieldNames, reader); initBatch(); }
public ReaderOptions filesystem(FileSystem fs) { super.filesystem(fs); return this; }
public ReaderOptions filesystem(FileSystem fs) { super.filesystem(fs); return this; }