@Override public void write(DataOutput out) throws IOException { if (codec != null) { for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } else { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } }
@Override public void write(DataOutput out) throws IOException { if (codec != null) { for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } else { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } }
/** * get number of bytes to store the keyBuffer. * * @return number of bytes used to store this KeyBuffer on disk * @throws IOException */ public int getSize() throws IOException { int ret = 0; ret += WritableUtils.getVIntSize(numberRows); for (int i = 0; i < eachColumnValueLen.length; i++) { ret += WritableUtils.getVIntSize(eachColumnValueLen[i]); ret += WritableUtils.getVIntSize(eachColumnUncompressedValueLen[i]); ret += WritableUtils.getVIntSize(allCellValLenBuffer[i].getLength()); ret += allCellValLenBuffer[i].getLength(); } return ret; }
/** * get number of bytes to store the keyBuffer. * * @return number of bytes used to store this KeyBuffer on disk * @throws IOException */ public int getSize() throws IOException { int ret = 0; ret += WritableUtils.getVIntSize(numberRows); for (int i = 0; i < eachColumnValueLen.length; i++) { ret += WritableUtils.getVIntSize(eachColumnValueLen[i]); ret += WritableUtils.getVIntSize(eachColumnUncompressedValueLen[i]); ret += WritableUtils.getVIntSize(allCellValLenBuffer[i].getLength()); ret += allCellValLenBuffer[i].getLength(); } return ret; }
@Override public void write(DataOutput out) throws IOException { // out.writeInt(numberRows); WritableUtils.writeVLong(out, numberRows); for (int i = 0; i < eachColumnValueLen.length; i++) { WritableUtils.writeVLong(out, eachColumnValueLen[i]); WritableUtils.writeVLong(out, eachColumnUncompressedValueLen[i]); NonSyncDataOutputBuffer colRowsLenBuf = allCellValLenBuffer[i]; int bufLen = colRowsLenBuf.getLength(); WritableUtils.writeVLong(out, bufLen); out.write(colRowsLenBuf.getData(), 0, bufLen); } }
@Override public void write(DataOutput out) throws IOException { // out.writeInt(numberRows); WritableUtils.writeVLong(out, numberRows); for (int i = 0; i < eachColumnValueLen.length; i++) { WritableUtils.writeVLong(out, eachColumnValueLen[i]); WritableUtils.writeVLong(out, eachColumnUncompressedValueLen[i]); NonSyncDataOutputBuffer colRowsLenBuf = allCellValLenBuffer[i]; int bufLen = colRowsLenBuf.getLength(); WritableUtils.writeVLong(out, bufLen); out.write(colRowsLenBuf.getData(), 0, bufLen); } }
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { try { barrStr.reset(); StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < numColumns; i++) { Object o = soi.getStructFieldData(obj, fields.get(i)); ObjectInspector oi = fields.get(i).getFieldObjectInspector(); serializeField(o, oi, row.get(i)); } // End of the record is part of the data tbOut.writeEndOfRecord(); serializeBytesWritable.set(barrStr.getData(), 0, barrStr.getLength()); } catch (IOException e) { throw new SerDeException(e.getMessage()); } return serializeBytesWritable; }
deflateOut.write(columnValue.getData(), 0, columnValue.getLength()); deflateOut.flush(); deflateFilter.finish(); colLen = valueBuffer.getLength() - valueLength; } else { colLen = columnValuePlainLength[columnIndex]; out.write(valueBuffer.getData(), 0, valueBuffer.getLength()); } else { for(int columnIndex=0; columnIndex < columnNumber; ++columnIndex) { NonSyncDataOutputBuffer buf = columnBuffers[columnIndex].columnValBuffer; out.write(buf.getData(), 0, buf.getLength());
deflateOut.write(columnValue.getData(), 0, columnValue.getLength()); deflateOut.flush(); deflateFilter.finish(); colLen = valueBuffer.getLength() - valueLength; } else { colLen = columnValuePlainLength[columnIndex]; out.write(valueBuffer.getData(), 0, valueBuffer.getLength()); } else { for(int columnIndex=0; columnIndex < columnNumber; ++columnIndex) { NonSyncDataOutputBuffer buf = columnBuffers[columnIndex].columnValBuffer; out.write(buf.getData(), 0, buf.getLength());
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
/** * Tests methods of {@link NonSyncDataInputBuffer}. * @throws IOException */ @Test public void testBaseBuffers() throws IOException { NonSyncDataOutputBuffer dob = new NonSyncDataOutputBuffer(); final Random r = new Random(); final long seed = 0x0123456789ABCDEFL; // hardcoded for reproducibility. r.setSeed(seed); System.out.println("SEED: " + seed); writeJunk(dob, r, seed, 1000); NonSyncDataInputBuffer dib = new NonSyncDataInputBuffer(); dib.reset(dob.getData(), 0, dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); dob.reset(); writeJunk(dob, r, seed, 1000); dib.reset(dob.getData(), dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); }
public void testReadAndWrite() throws IOException { String testString = "test_hive_input_output_number_0"; byte[] string_bytes = testString.getBytes(); NonSyncDataInputBuffer inBuffer = new NonSyncDataInputBuffer(); NonSyncDataOutputBuffer outBuffer = new NonSyncDataOutputBuffer(); try { outBuffer.write(string_bytes); inBuffer.reset(outBuffer.getData(), 0, outBuffer.getLength()); byte[] readBytes = new byte[string_bytes.length]; inBuffer.read(readBytes); String readString = new String(readBytes); assertEquals("Field testReadAndWrite()", readString, testString); } finally { inBuffer.close(); outBuffer.close(); } }
.getData(), currentKey.allCellValLenBuffer[columnID].getLength()); SelectedColumn selCol = selectedColumns[selColIdx]; byte[] uncompData = null;
int colIx = col.colIndex; NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); col.rowReadIndex = 0; col.runLength = 0;
.getData(), currentKey.allCellValLenBuffer[columnID].getLength()); SelectedColumn selCol = selectedColumns[selColIdx]; byte[] uncompData = null;
int colIx = col.colIndex; NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); col.rowReadIndex = 0; col.runLength = 0;
@Override public void write(DataOutput out) throws IOException { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } }
@Override public void write(DataOutput out) throws IOException { if (codec != null) { for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } else { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } }