writer.close(); RCFile.Reader reader = new RCFile.Reader(fs, file, conf); assertTrue(reader.next(rowID)); assertEquals(rowID.get(), 0L); assertTrue(reader.next(rowID)); assertEquals(rowID.get(), 1L); BytesRefWritable brw; for (int col=0; col < 8; col++) { BytesRefArrayWritable result2 = reader.getColumn(col, result); if (result == null) { assertNotNull(result2); reader.close();
new Text("hive and hadoop"), null, null}; RCFile.Reader reader = new RCFile.Reader(fs, file, conf); assertEquals(new Text("block"), reader.getMetadata().get(new Text("apple"))); assertEquals(new Text("block"), reader.getMetadataValueOf(new Text("apple"))); assertEquals(new Text("dog"), reader.getMetadataValueOf(new Text("cat"))); LongWritable rowID = new LongWritable(); reader.next(rowID); BytesRefArrayWritable cols = new BytesRefArrayWritable(); reader.getCurrentRow(cols); cols.resetValid(8); Object row = serDe.deserialize(cols); reader.close();
public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass, RecordReader recordReader) throws IOException { boolean blockPointer = false; long blockStart = -1; FileSplit fileSplit = split; Path path = fileSplit.getPath(); FileSystem fs = path.getFileSystem(job); if (inputFormatClass.getName().contains("SequenceFile")) { SequenceFile.Reader in = new SequenceFile.Reader(fs, path, job); blockPointer = in.isBlockCompressed(); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } else if (recordReader instanceof RCFileRecordReader) { blockPointer = true; blockStart = ((RCFileRecordReader) recordReader).getStart(); } else if (inputFormatClass.getName().contains("RCFile")) { blockPointer = true; RCFile.Reader in = new RCFile.Reader(fs, path, job); in.sync(fileSplit.getStart()); blockStart = in.getPosition(); in.close(); } this.jobConf = job; this.initIOContext(blockStart, blockPointer, path.makeQualified(fs)); this.initIOContextSortedProps(split, recordReader, job); }
conf.setInt("io.file.buffer.size", bufferSize); this.file = file; in = openFile(fs, file, bufferSize, length); this.conf = conf; end = start + length; try { if (start > 0) { seek(0); init(); seek(start); } else { init(); currentKey = createKeyBuffer(); boolean lazyDecompress = !tolerateCorruptions; currentValue = new ValueBuffer(
conf.setInt("io.file.buffer.size", bufferSize); this.file = file; in = openFile(fs, file, bufferSize, length); this.conf = conf; end = start + length; try { if (start > 0) { seek(0); init(); seek(start); } else { init(); currentKey = createKeyBuffer(); currentValue = new ValueBuffer(null, columnNumber, skippedColIDs, codec);
currentValueBuffer(); currentValueBuffer(); BytesRefWritable ref = ret.unCheckedGet(i); colAdvanceRow(j, col); BytesRefWritable ref = ret.unCheckedGet(i); colAdvanceRow(j, col); ref.set(currentValue.loadedColumnsValueBuffer[j].getData(), col.rowReadIndex, col.prvLength);
if (!recordReader.nextColumnsBatch() || recordReader.lastSeenSyncPos() >= endFilePosition) { close(); return null; currentBatchSize = recordReader.getCurrentKeyBufferObj().getNumberRows(); positionInBatch = 0; completedBytes = recordReader.getPosition() - startFilePosition;
currentValueBuffer(); colAdvanceRow(selColIdx, selCol); int length = selCol.prvLength;
final BytesRefArrayWritable cols = new BytesRefArrayWritable(); while (reader.next(key)) { System.out.println("Getting next row."); reader.getCurrentRow(cols);
currentValueBuffer(); .getData(), currentKey.allCellValLenBuffer[columnID].getLength()); for (int i = 0; i < recordsNumInValBuffer; i++) { int length = getColumnNextValueLength(columnID);
addPartitionCols = false; in.getCurrentRow(colsCache);
currentValueBuffer(); int length = getColumnNextValueLength(i); columnRowReadIndex[i] = columnCurrentRowStart + length;
boolean next; try { next = reader.next(rowID); } catch (EOFException ex) { BytesRefArrayWritable rowBatchBytes = new BytesRefArrayWritable(); rowBatchBytes.resetValid(columns.size()); reader.getCurrentRow(rowBatchBytes);
boolean next; try { next = reader.next(rowID); } catch (EOFException ex) { BytesRefArrayWritable rowBatchBytes = new BytesRefArrayWritable(); rowBatchBytes.resetValid(columns.size()); reader.getCurrentRow(rowBatchBytes);
private int nextKeyValueTolerateCorruptions() throws IOException { long currentOffset = in.getPos(); int ret; try { ret = nextKeyBuffer(); this.currentValueBuffer(); } catch (IOException ioe) { // A BlockMissingException indicates a temporary error, // not a corruption. Re-throw this exception. String msg = ioe.getMessage(); if (msg != null && msg.startsWith(BLOCK_MISSING_MESSAGE)) { LOG.warn("Re-throwing block-missing exception" + ioe); throw ioe; } // We have an IOException other than a BlockMissingException. LOG.warn("Ignoring IOException in file " + file + " after offset " + currentOffset, ioe); ret = -1; } catch (Throwable t) { // We got an exception that is not IOException // (typically OOM, IndexOutOfBounds, InternalError). // This is most likely a corruption. LOG.warn("Ignoring unknown error in " + file + " after offset " + currentOffset, t); ret = -1; } return ret; }
protected boolean nextBlock(RCFileKeyBufferWrapper keyWrapper, RCFileValueBufferWrapper valueWrapper) throws IOException { if (!more) { return false; } more = in.nextBlock(); if (!more) { return false; } keyWrapper.keyBuffer = this.in.getCurrentKeyBufferObj(); keyWrapper.recordLength = this.in.getCurrentBlockLength(); keyWrapper.keyLength = this.in.getCurrentKeyLength(); keyWrapper.compressedKeyLength = this.in.getCurrentCompressedKeyLen(); keyWrapper.codec = this.in.getCompressionCodec(); keyWrapper.inputPath = path; valueWrapper.valueBuffer = this.in.getCurrentValueBufferObj(); long lastSeenSyncPos = in.lastSeenSyncPos(); if (lastSeenSyncPos >= end) { more = false; return more; } return more; }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { FileSplit fSplit = (FileSplit) split; Path path = fSplit.getPath(); Configuration conf = context.getConfiguration(); this.in = new RCFile.Reader(path.getFileSystem(conf), path, conf); this.end = fSplit.getStart() + fSplit.getLength(); if (fSplit.getStart() > in.getPosition()) { in.sync(fSplit.getStart()); } this.start = in.getPosition(); more = start < end; key = new LongWritable(); value = new BytesRefArrayWritable(); } }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { FileSplit fSplit = (FileSplit) split; Path path = fSplit.getPath(); Configuration conf = context.getConfiguration(); this.in = new RCFile.Reader(path.getFileSystem(conf), path, conf); this.end = fSplit.getStart() + fSplit.getLength(); if (fSplit.getStart() > in.getPosition()) { in.sync(fSplit.getStart()); } this.start = in.getPosition(); more = start < end; key = new LongWritable(); value = new BytesRefArrayWritable(); } }
@Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException, InterruptedException { FileSplit fSplit = (FileSplit) split; Path path = fSplit.getPath(); Configuration conf = context.getConfiguration(); this.in = new RCFile.Reader(path.getFileSystem(conf), path, conf); this.end = fSplit.getStart() + fSplit.getLength(); if (fSplit.getStart() > in.getPosition()) { in.sync(fSplit.getStart()); } this.start = in.getPosition(); more = start < end; key = new LongWritable(); value = new BytesRefArrayWritable(); } }
protected boolean next(LongWritable key) throws IOException { if (!more) { return false; } more = in.next(key); long lastSeenSyncPos = in.lastSeenSyncPos(); if (lastSeenSyncPos >= end) { if (useCache) { syncCache.put(split, lastSeenSyncPos); } more = false; return more; } return more; }