public void nullColumn(int columnIndex) { eachColumnValueLen[columnIndex] = 0; eachColumnUncompressedValueLen[columnIndex] = 0; allCellValLenBuffer[columnIndex] = new NonSyncDataOutputBuffer(); }
ColumnBuffer() throws IOException { columnValBuffer = new NonSyncDataOutputBuffer(); valLenBuffer = new NonSyncDataOutputBuffer(); }
public void nullColumn(int columnIndex) { eachColumnValueLen[columnIndex] = 0; eachColumnUncompressedValueLen[columnIndex] = 0; allCellValLenBuffer[columnIndex] = new NonSyncDataOutputBuffer(); }
ColumnBuffer() throws IOException { columnValBuffer = new NonSyncDataOutputBuffer(); valLenBuffer = new NonSyncDataOutputBuffer(); }
barrStr = new NonSyncDataOutputBuffer(); tbOut = new TypedBytesWritableOutput(barrStr);
continue; loadedColumnsValueBuffer[readIndex] = new NonSyncDataOutputBuffer(); if (codec != null) { decompressedFlag[readIndex] = false; lazyDecompressCallbackObjs[readIndex] = new LazyDecompressionCallbackImpl( readIndex, k); compressedColumnsValueBuffer[readIndex] = new NonSyncDataOutputBuffer(); } else { decompressedFlag[readIndex] = true;
continue; loadedColumnsValueBuffer[readIndex] = new NonSyncDataOutputBuffer(); if (codec != null) { decompressedFlag[readIndex] = false; lazyDecompressCallbackObjs[readIndex] = new LazyDecompressionCallbackImpl( readIndex, k); compressedColumnsValueBuffer[readIndex] = new NonSyncDataOutputBuffer(); } else { decompressedFlag[readIndex] = true;
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
ReflectionUtils.setConf(codec, this.conf); compressor = CodecPool.getCompressor(codec); valueBuffer = new NonSyncDataOutputBuffer(); deflateFilter = codec.createOutputStream(valueBuffer, compressor); deflateOut = new DataOutputStream(deflateFilter);
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
ReflectionUtils.setConf(codec, this.conf); compressor = CodecPool.getCompressor(codec); valueBuffer = new NonSyncDataOutputBuffer(); deflateFilter = codec.createOutputStream(valueBuffer, compressor); deflateOut = new DataOutputStream(deflateFilter);
public void testReadAndWrite() throws IOException { String testString = "test_hive_input_output_number_0"; byte[] string_bytes = testString.getBytes(); NonSyncDataInputBuffer inBuffer = new NonSyncDataInputBuffer(); NonSyncDataOutputBuffer outBuffer = new NonSyncDataOutputBuffer(); try { outBuffer.write(string_bytes); inBuffer.reset(outBuffer.getData(), 0, outBuffer.getLength()); byte[] readBytes = new byte[string_bytes.length]; inBuffer.read(readBytes); String readString = new String(readBytes); assertEquals("Field testReadAndWrite()", readString, testString); } finally { inBuffer.close(); outBuffer.close(); } }
/** * Tests methods of {@link NonSyncDataInputBuffer}. * @throws IOException */ @Test public void testBaseBuffers() throws IOException { NonSyncDataOutputBuffer dob = new NonSyncDataOutputBuffer(); final Random r = new Random(); final long seed = 0x0123456789ABCDEFL; // hardcoded for reproducibility. r.setSeed(seed); System.out.println("SEED: " + seed); writeJunk(dob, r, seed, 1000); NonSyncDataInputBuffer dib = new NonSyncDataInputBuffer(); dib.reset(dob.getData(), 0, dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); dob.reset(); writeJunk(dob, r, seed, 1000); dib.reset(dob.getData(), dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); }
public void nullColumn(int columnIndex) { eachColumnValueLen[columnIndex] = 0; eachColumnUncompressedValueLen[columnIndex] = 0; allCellValLenBuffer[columnIndex] = new NonSyncDataOutputBuffer(); }
ColumnBuffer() throws IOException { columnValBuffer = new NonSyncDataOutputBuffer(); valLenBuffer = new NonSyncDataOutputBuffer(); }
ColumnBuffer() throws IOException { columnValBuffer = new NonSyncDataOutputBuffer(); valLenBuffer = new NonSyncDataOutputBuffer(); }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }