@Override public void write(DataOutput out) throws IOException { if (codec != null) { for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } else { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } }
@Override public void write(DataOutput out) throws IOException { if (codec != null) { for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } else { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
@Override public void write(DataOutput out) throws IOException { // out.writeInt(numberRows); WritableUtils.writeVLong(out, numberRows); for (int i = 0; i < eachColumnValueLen.length; i++) { WritableUtils.writeVLong(out, eachColumnValueLen[i]); WritableUtils.writeVLong(out, eachColumnUncompressedValueLen[i]); NonSyncDataOutputBuffer colRowsLenBuf = allCellValLenBuffer[i]; int bufLen = colRowsLenBuf.getLength(); WritableUtils.writeVLong(out, bufLen); out.write(colRowsLenBuf.getData(), 0, bufLen); } }
@Override public void write(DataOutput out) throws IOException { // out.writeInt(numberRows); WritableUtils.writeVLong(out, numberRows); for (int i = 0; i < eachColumnValueLen.length; i++) { WritableUtils.writeVLong(out, eachColumnValueLen[i]); WritableUtils.writeVLong(out, eachColumnUncompressedValueLen[i]); NonSyncDataOutputBuffer colRowsLenBuf = allCellValLenBuffer[i]; int bufLen = colRowsLenBuf.getLength(); WritableUtils.writeVLong(out, bufLen); out.write(colRowsLenBuf.getData(), 0, bufLen); } }
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { try { barrStr.reset(); StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < numColumns; i++) { Object o = soi.getStructFieldData(obj, fields.get(i)); ObjectInspector oi = fields.get(i).getFieldObjectInspector(); serializeField(o, oi, row.get(i)); } // End of the record is part of the data tbOut.writeEndOfRecord(); serializeBytesWritable.set(barrStr.getData(), 0, barrStr.getLength()); } catch (IOException e) { throw new SerDeException(e.getMessage()); } return serializeBytesWritable; }
deflateOut.write(columnValue.getData(), 0, columnValue.getLength()); deflateOut.flush(); deflateFilter.finish(); out.write(valueBuffer.getData(), 0, valueBuffer.getLength()); } else { for(int columnIndex=0; columnIndex < columnNumber; ++columnIndex) { NonSyncDataOutputBuffer buf = columnBuffers[columnIndex].columnValBuffer; out.write(buf.getData(), 0, buf.getLength());
keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( keyDecompressBuffer, keyDecompressor); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn); } else { int colIx = col.colIndex; NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); col.rowReadIndex = 0; col.runLength = 0;
tbOut.writeEndOfRecord(); if (barrStr.getLength() > 0) { ((BytesWritable) data).set(barrStr.getData(), 0, barrStr.getLength());
deflateOut.write(columnValue.getData(), 0, columnValue.getLength()); deflateOut.flush(); deflateFilter.finish(); out.write(valueBuffer.getData(), 0, valueBuffer.getLength()); } else { for(int columnIndex=0; columnIndex < columnNumber; ++columnIndex) { NonSyncDataOutputBuffer buf = columnBuffers[columnIndex].columnValBuffer; out.write(buf.getData(), 0, buf.getLength());
.getData(), currentKey.allCellValLenBuffer[columnID].getLength()); SelectedColumn selCol = selectedColumns[selColIdx]; byte[] uncompData = null; if (decompressed) { uncompData = currentValue.loadedColumnsValueBuffer[selColIdx].getData(); } else { decompCallBack = currentValue.lazyDecompressCallbackObjs[selColIdx];
.getData(), currentKey.allCellValLenBuffer[columnID].getLength()); SelectedColumn selCol = selectedColumns[selColIdx]; byte[] uncompData = null; if (decompressed) { uncompData = currentValue.loadedColumnsValueBuffer[selColIdx].getData(); } else { decompCallBack = currentValue.lazyDecompressCallbackObjs[selColIdx];
keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( keyDecompressBuffer, keyDecompressor); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn); } else { int colIx = col.colIndex; NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); col.rowReadIndex = 0; col.runLength = 0;
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
public void testReadAndWrite() throws IOException { String testString = "test_hive_input_output_number_0"; byte[] string_bytes = testString.getBytes(); NonSyncDataInputBuffer inBuffer = new NonSyncDataInputBuffer(); NonSyncDataOutputBuffer outBuffer = new NonSyncDataOutputBuffer(); try { outBuffer.write(string_bytes); inBuffer.reset(outBuffer.getData(), 0, outBuffer.getLength()); byte[] readBytes = new byte[string_bytes.length]; inBuffer.read(readBytes); String readString = new String(readBytes); assertEquals("Field testReadAndWrite()", readString, testString); } finally { inBuffer.close(); outBuffer.close(); } }
/** * Tests methods of {@link NonSyncDataInputBuffer}. * @throws IOException */ @Test public void testBaseBuffers() throws IOException { NonSyncDataOutputBuffer dob = new NonSyncDataOutputBuffer(); final Random r = new Random(); final long seed = 0x0123456789ABCDEFL; // hardcoded for reproducibility. r.setSeed(seed); System.out.println("SEED: " + seed); writeJunk(dob, r, seed, 1000); NonSyncDataInputBuffer dib = new NonSyncDataInputBuffer(); dib.reset(dob.getData(), 0, dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); dob.reset(); writeJunk(dob, r, seed, 1000); dib.reset(dob.getData(), dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); }