valBuf.write(in, vaRowsLen); if (codec != null) { decompressedFlag[addIndex] = false;
valBuf.write(in, vaRowsLen); if (codec != null) { decompressedFlag[addIndex] = false;
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
if (decompress) { keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( deflatFilter.resetState(); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn);
public void testReadAndWrite() throws IOException { String testString = "test_hive_input_output_number_0"; byte[] string_bytes = testString.getBytes(); NonSyncDataInputBuffer inBuffer = new NonSyncDataInputBuffer(); NonSyncDataOutputBuffer outBuffer = new NonSyncDataOutputBuffer(); try { outBuffer.write(string_bytes); inBuffer.reset(outBuffer.getData(), 0, outBuffer.getLength()); byte[] readBytes = new byte[string_bytes.length]; inBuffer.read(readBytes); String readString = new String(readBytes); assertEquals("Field testReadAndWrite()", readString, testString); } finally { inBuffer.close(); outBuffer.close(); } }
if (decompress) { keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( deflatFilter.resetState(); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn);
@Override public void readFields(DataInput in) throws IOException { int addIndex = 0; int skipTotal = 0; for (int i = 0; i < columnNumber; i++) { int vaRowsLen = keyBuffer.eachColumnValueLen[i]; // skip this column if (skippedColIDs[i]) { skipTotal += vaRowsLen; continue; } if (skipTotal != 0) { in.skipBytes(skipTotal); skipTotal = 0; } NonSyncDataOutputBuffer valBuf = loadedColumnsValueBuffer[addIndex]; valBuf.reset(); valBuf.write(in, vaRowsLen); if (codec != null) { decompressedFlag[addIndex] = false; } addIndex++; } if (skipTotal != 0) { in.skipBytes(skipTotal); } }
valBuf.write(in, vaRowsLen); if (codec != null) { decompressedFlag[addIndex] = false;
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = loadedColumnsValueBuffer[index]; NonSyncDataOutputBuffer decompressedData = new NonSyncDataOutputBuffer(); decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); decompressedData.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); loadedColumnsValueBuffer[index] = decompressedData; decompressedFlag[index] = true; return decompressedData.getData(); } }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
if (decompress) { keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( deflatFilter.resetState(); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn);
if (decompress) { keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( deflatFilter.resetState(); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn);