public RecordReader<NullWritable, VectorizedRowBatch> checkLlapSplit( InputSplit split, JobConf job, Reporter reporter) throws IOException { boolean useLlapIo = true; if (split instanceof LlapAwareSplit) { useLlapIo = ((LlapAwareSplit) split).canUseLlapIo(job); } if (useLlapIo) return null; LlapIoImpl.LOG.warn("Not using LLAP IO for an unsupported split: " + split); return sourceInputFormat.getRecordReader(split, job, reporter); }
public RecordReader<WritableComparable, Writable> getRecordReader(JobConf job) throws IOException { return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL); } }
public RecordReader<WritableComparable, Writable> getRecordReader(JobConf job) throws IOException { return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL); } }
protected org.apache.hadoop.mapred.RecordReader setReaderAtSplit(int splitNum) throws IOException { JobConf localJc = getLocalFSJobConfClone(jc); currentSplitPointer = splitNum; if ( rr != null ) { rr.close(); } // open record reader to read next split rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], jobCloneUsingLocalFs, reporter); currentSplitPointer++; return rr; }
protected org.apache.hadoop.mapred.RecordReader setReaderAtSplit(int splitNum) throws IOException { JobConf localJc = getLocalFSJobConfClone(jc); currentSplitPointer = splitNum; if ( rr != null ) { rr.close(); } // open record reader to read next split rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], jobCloneUsingLocalFs, reporter); currentSplitPointer++; return rr; }
RecordReader<WritableComparable, Writable> recordReader = (RecordReader<WritableComparable, Writable>) inputFormat.getRecordReader(fileSplit, jobConf, Reporter.NULL);
/** * Initializes next reader if available, will close previous reader if any. * * @param job map / reduce job configuration. * @return true if new reader was initialized, false is no more readers are available * @throws ExecutionSetupException if could not init record reader */ protected boolean initNextReader(JobConf job) throws ExecutionSetupException { if (inputSplitsIterator.hasNext()) { if (reader != null) { closeReader(); } InputSplit inputSplit = inputSplitsIterator.next(); try { reader = (org.apache.hadoop.mapred.RecordReader<Object, Object>) job.getInputFormat().getRecordReader(inputSplit, job, Reporter.NULL); logger.trace("hive reader created: {} for inputSplit {}", reader.getClass().getName(), inputSplit.toString()); } catch (Exception e) { throw new ExecutionSetupException("Failed to get o.a.hadoop.mapred.RecordReader from Hive InputFormat", e); } return true; } return false; }
@SuppressWarnings("unchecked") public RecordReader<K, V> getRecordReader(InputSplit split, JobConf conf, Reporter reporter) throws IOException { // Find the Schema and then build the RecordReader from the // TaggedInputSplit. TaggedInputSplit taggedInputSplit = (TaggedInputSplit) split; Schema schema = taggedInputSplit.getSchema(); AvroJob.setInputSchema(conf, schema); InputFormat<K, V> inputFormat = (InputFormat<K, V>) ReflectionUtils .newInstance(taggedInputSplit.getInputFormatClass(), conf); return inputFormat.getRecordReader(taggedInputSplit.getInputSplit(), conf, reporter); } }
@Override public void open(HadoopInputSplit split) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { this.recordReader = this.mapredInputFormat.getRecordReader(split.getHadoopInputSplit(), jobConf, new HadoopDummyReporter()); if (this.recordReader instanceof Configurable) { ((Configurable) this.recordReader).setConf(jobConf); } key = this.recordReader.createKey(); value = this.recordReader.createValue(); this.fetched = false; } }
JobConf localJc = getLocalFSJobConfClone(jc); rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], jobCloneUsingLocalFs, reporter); currentSplitPointer++;
if (rr == null) { return sourceInputFormat.getRecordReader(split, job, reporter); return sourceInputFormat.getRecordReader(split, job, reporter);
private org.apache.hadoop.mapred.RecordReader createBaseRecordReader(HCatSplit hcatSplit, HiveStorageHandler storageHandler, TaskAttemptContext taskContext) throws IOException { JobConf jobConf = HCatUtil.getJobConfFromContext(taskContext); HCatUtil.copyJobPropertiesToJobConf(hcatSplit.getPartitionInfo().getJobProperties(), jobConf); org.apache.hadoop.mapred.InputFormat inputFormat = HCatInputFormat.getMapRedInputFormat(jobConf, storageHandler.getInputFormatClass()); return inputFormat.getRecordReader(hcatSplit.getBaseSplit(), jobConf, InternalUtil.createReporter(taskContext)); }
RecordReader<K, V> recordReader = inputFormat.getRecordReader( new FileSplit(new Path(tempFile.getFile().getAbsolutePath()), 0, tempFile.getFile().length(), (String[]) null), configuration,
rr = inputFormat.getRecordReader(inputSplits[currentSplitPointer], localJc, reporter); currentSplitPointer++;
/** * Get the record reader for the next chunk in this * BucketizedHiveRecordReader. */ protected boolean initNextRecordReader() throws IOException { if (recordReader != null) { recordReader.close(); recordReader = null; if (idx > 0) { progress += split.getLength(idx - 1); // done processing so far } } // if all chunks have been processed, nothing more to do. if (idx == split.getNumSplits()) { return false; } // get a record reader for the idx-th chunk try { recordReader = inputFormat.getRecordReader(split.getSplit(idx), jobConf, reporter); } catch (Exception e) { recordReader = HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(e, jobConf); } // if we're performing a binary search, we need to restart it if (isSorted) { initIOContextSortedProps((FileSplit) split.getSplit(idx), recordReader, jobConf); } idx++; return true; }
/** * Get the record reader for the next chunk in this * BucketizedHiveRecordReader. */ protected boolean initNextRecordReader() throws IOException { if (recordReader != null) { recordReader.close(); recordReader = null; if (idx > 0) { progress += split.getLength(idx - 1); // done processing so far } } // if all chunks have been processed, nothing more to do. if (idx == split.getNumSplits()) { return false; } // get a record reader for the idx-th chunk try { recordReader = inputFormat.getRecordReader(split.getSplit(idx), jobConf, reporter); } catch (Exception e) { recordReader = HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(e, jobConf); } // if we're performing a binary search, we need to restart it if (isSorted) { initIOContextSortedProps((FileSplit) split.getSplit(idx), recordReader, jobConf); } idx++; return true; }
for(InputSplit is : splits) { final AcidRecordReader<NullWritable, OrcStruct> recordReader = (AcidRecordReader<NullWritable, OrcStruct>) inputFormat .getRecordReader(is, job, Reporter.NULL);
ReaderWithOffsets offsetReader = null; @SuppressWarnings("rawtypes") RecordReader sourceReader = sourceInputFormat.getRecordReader(split, jobConf, reporter); try { offsetReader = createOffsetReader(sourceReader);
Assert.assertEquals(numExpectedFiles, splits.length); org.apache.hadoop.mapred.RecordReader<NullWritable, OrcStruct> rr = inf.getRecordReader(splits[0], job, Reporter.NULL);
.getLocations()); this.setRecordReader(inputFormat.getRecordReader(fsplit, jobConf, reporter));