ReflectionUtils.setConf(codec, this.conf); compressor = CodecPool.getCompressor(codec); valueBuffer = new NonSyncDataOutputBuffer(); deflateFilter = codec.createOutputStream(valueBuffer, compressor); deflateOut = new DataOutputStream(deflateFilter); deflateOut.write(columnValue.getData(), 0, columnValue.getLength()); deflateOut.flush(); deflateFilter.finish(); colLen = valueBuffer.getLength() - valueLength; } else { colLen = columnValuePlainLength[columnIndex]; out.write(valueBuffer.getData(), 0, valueBuffer.getLength()); } else { for(int columnIndex=0; columnIndex < columnNumber; ++columnIndex) { NonSyncDataOutputBuffer buf = columnBuffers[columnIndex].columnValBuffer; out.write(buf.getData(), 0, buf.getLength());
@Override public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException { try { barrStr.reset(); StructObjectInspector soi = (StructObjectInspector) objInspector; List<? extends StructField> fields = soi.getAllStructFieldRefs(); for (int i = 0; i < numColumns; i++) { Object o = soi.getStructFieldData(obj, fields.get(i)); ObjectInspector oi = fields.get(i).getFieldObjectInspector(); serializeField(o, oi, row.get(i)); } // End of the record is part of the data tbOut.writeEndOfRecord(); serializeBytesWritable.set(barrStr.getData(), 0, barrStr.getLength()); } catch (IOException e) { throw new SerDeException(e.getMessage()); } return serializeBytesWritable; }
compressedKeyLen = in.readInt(); if (decompress) { keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( keyDecompressBuffer, keyDecompressor); DataInputStream compressedIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn); } else { int colIx = col.colIndex; NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); col.rowReadIndex = 0; col.runLength = 0;
@Override public void write(DataOutput out) throws IOException { if (codec != null) { for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } else { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
public void testReadAndWrite() throws IOException { String testString = "test_hive_input_output_number_0"; byte[] string_bytes = testString.getBytes(); NonSyncDataInputBuffer inBuffer = new NonSyncDataInputBuffer(); NonSyncDataOutputBuffer outBuffer = new NonSyncDataOutputBuffer(); try { outBuffer.write(string_bytes); inBuffer.reset(outBuffer.getData(), 0, outBuffer.getLength()); byte[] readBytes = new byte[string_bytes.length]; inBuffer.read(readBytes); String readString = new String(readBytes); assertEquals("Field testReadAndWrite()", readString, testString); } finally { inBuffer.close(); outBuffer.close(); } }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
public void nullColumn(int columnIndex) { eachColumnValueLen[columnIndex] = 0; eachColumnUncompressedValueLen[columnIndex] = 0; allCellValLenBuffer[columnIndex] = new NonSyncDataOutputBuffer(); }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = loadedColumnsValueBuffer[index]; NonSyncDataOutputBuffer decompressedData = new NonSyncDataOutputBuffer(); decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); decompressedData.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); loadedColumnsValueBuffer[index] = decompressedData; decompressedFlag[index] = true; return decompressedData.getData(); } }
/** * get number of bytes to store the keyBuffer. * * @return number of bytes used to store this KeyBuffer on disk * @throws IOException */ public int getSize() throws IOException { int ret = 0; ret += WritableUtils.getVIntSize(numberRows); for (int i = 0; i < eachColumnValueLen.length; i++) { ret += WritableUtils.getVIntSize(eachColumnValueLen[i]); ret += WritableUtils.getVIntSize(eachColumnUncompressedValueLen[i]); ret += WritableUtils.getVIntSize(allCellValLenBuffer[i].getLength()); ret += allCellValLenBuffer[i].getLength(); } return ret; }
@Override public void write(int b) throws IOException { buffer.write(b); incCount(1); }
private void writeKey(KeyBuffer keyBuffer, int recordLen, int keyLength) throws IOException { checkAndWriteSync(); // sync out.writeInt(recordLen); // total record length out.writeInt(keyLength); // key portion length if(this.isCompressed()) { Compressor compressor = CodecPool.getCompressor(codec); NonSyncDataOutputBuffer compressionBuffer = new NonSyncDataOutputBuffer(); CompressionOutputStream deflateFilter = codec.createOutputStream(compressionBuffer, compressor); DataOutputStream deflateOut = new DataOutputStream(deflateFilter); //compress key and write key out compressionBuffer.reset(); deflateFilter.resetState(); keyBuffer.write(deflateOut); deflateOut.flush(); deflateFilter.finish(); int compressedKeyLen = compressionBuffer.getLength(); out.writeInt(compressedKeyLen); out.write(compressionBuffer.getData(), 0, compressedKeyLen); CodecPool.returnCompressor(compressor); } else { out.writeInt(keyLength); keyBuffer.write(out); } }
@Override public void write(DataOutput out) throws IOException { if (codec != null) { for (NonSyncDataOutputBuffer currentBuf : compressedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } else { for (NonSyncDataOutputBuffer currentBuf : loadedColumnsValueBuffer) { out.write(currentBuf.getData(), 0, currentBuf.getLength()); } } }
@Override public void readFields(DataInput in) throws IOException { eachColumnValueLen = new int[columnNumber]; eachColumnUncompressedValueLen = new int[columnNumber]; allCellValLenBuffer = new NonSyncDataOutputBuffer[columnNumber]; numberRows = WritableUtils.readVInt(in); for (int i = 0; i < columnNumber; i++) { eachColumnValueLen[i] = WritableUtils.readVInt(in); eachColumnUncompressedValueLen[i] = WritableUtils.readVInt(in); int bufLen = WritableUtils.readVInt(in); if (allCellValLenBuffer[i] == null) { allCellValLenBuffer[i] = new NonSyncDataOutputBuffer(); } else { allCellValLenBuffer[i].reset(); } allCellValLenBuffer[i].write(in, bufLen); } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
ColumnBuffer() throws IOException { columnValBuffer = new NonSyncDataOutputBuffer(); valLenBuffer = new NonSyncDataOutputBuffer(); }