public void clearColumnBuffer() throws IOException { decompressBuffer.reset(); }
public void clearColumnBuffer() throws IOException { decompressBuffer.reset(); }
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.getBytes(), 0, data.getLength()); try { for (int i = 0; i < columnNames.size(); i++) { row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( keyDecompressBuffer, keyDecompressor); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn); } else { int colIx = col.colIndex; NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); col.rowReadIndex = 0; col.runLength = 0;
keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( keyDecompressBuffer, keyDecompressor); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn); } else { int colIx = col.colIndex; NonSyncDataOutputBuffer buf = currentKey.allCellValLenBuffer[colIx]; colValLenBufferReadIn[selIx].reset(buf.getData(), buf.getLength()); col.rowReadIndex = 0; col.runLength = 0;
public void testReadAndWrite() throws IOException { String testString = "test_hive_input_output_number_0"; byte[] string_bytes = testString.getBytes(); NonSyncDataInputBuffer inBuffer = new NonSyncDataInputBuffer(); NonSyncDataOutputBuffer outBuffer = new NonSyncDataOutputBuffer(); try { outBuffer.write(string_bytes); inBuffer.reset(outBuffer.getData(), 0, outBuffer.getLength()); byte[] readBytes = new byte[string_bytes.length]; inBuffer.read(readBytes); String readString = new String(readBytes); assertEquals("Field testReadAndWrite()", readString, testString); } finally { inBuffer.close(); outBuffer.close(); } }
fetchColumnTempBuf.reset(currentKey.allCellValLenBuffer[columnID] .getData(), currentKey.allCellValLenBuffer[columnID].getLength()); SelectedColumn selCol = selectedColumns[selColIdx];
/** * Tests methods of {@link NonSyncDataInputBuffer}. * @throws IOException */ @Test public void testBaseBuffers() throws IOException { NonSyncDataOutputBuffer dob = new NonSyncDataOutputBuffer(); final Random r = new Random(); final long seed = 0x0123456789ABCDEFL; // hardcoded for reproducibility. r.setSeed(seed); System.out.println("SEED: " + seed); writeJunk(dob, r, seed, 1000); NonSyncDataInputBuffer dib = new NonSyncDataInputBuffer(); dib.reset(dob.getData(), 0, dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); dob.reset(); writeJunk(dob, r, seed, 1000); dib.reset(dob.getData(), dob.getLength()); assertEquals(0, dib.getPosition()); assertEquals(dob.getLength(), dib.getLength()); readJunk(dib, r, seed, 1000); }
fetchColumnTempBuf.reset(currentKey.allCellValLenBuffer[columnID] .getData(), currentKey.allCellValLenBuffer[columnID].getLength()); SelectedColumn selCol = selectedColumns[selColIdx];
public void clearColumnBuffer() throws IOException { decompressBuffer.reset(); }
public void clearColumnBuffer() throws IOException { decompressBuffer.reset(); }
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.getBytes(), 0, data.getLength()); try { for (int i = 0; i < columnNames.size(); i++) { row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.getBytes(), 0, data.getLength()); try { for (int i = 0; i < columnNames.size(); i++) { row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.getBytes(), 0, data.getLength()); try { for (int i = 0; i < columnNames.size(); i++) { row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }
@Override public Object deserialize(Writable blob) throws SerDeException { BytesWritable data = (BytesWritable) blob; inBarrStr.reset(data.get(), 0, data.getSize()); try { for (int i = 0; i < columnNames.size(); i++) { row.set(i, deserializeField(tbIn, columnTypes.get(i), row.get(i))); } // The next byte should be the marker assert tbIn.readTypeCode() == Type.ENDOFRECORD; } catch (IOException e) { throw new SerDeException(e); } return row; }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = loadedColumnsValueBuffer[index]; NonSyncDataOutputBuffer decompressedData = new NonSyncDataOutputBuffer(); decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); decompressedData.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); loadedColumnsValueBuffer[index] = decompressedData; decompressedFlag[index] = true; return decompressedData.getData(); } }
@Override public byte[] decompress() throws IOException { if (decompressedFlag[index] || codec == null) { return loadedColumnsValueBuffer[index].getData(); } NonSyncDataOutputBuffer compressedData = compressedColumnsValueBuffer[index]; decompressBuffer.reset(); DataInputStream valueIn = new DataInputStream(deflatFilter); deflatFilter.resetState(); if (deflatFilter instanceof SchemaAwareCompressionInputStream) { ((SchemaAwareCompressionInputStream)deflatFilter).setColumnIndex(colIndex); } decompressBuffer.reset(compressedData.getData(), keyBuffer.eachColumnValueLen[colIndex]); NonSyncDataOutputBuffer decompressedColBuf = loadedColumnsValueBuffer[index]; decompressedColBuf.reset(); decompressedColBuf.write(valueIn, keyBuffer.eachColumnUncompressedValueLen[colIndex]); decompressedFlag[index] = true; numCompressed--; return decompressedColBuf.getData(); } }
keyTempBuffer.reset(); keyTempBuffer.write(in, compressedKeyLen); keyDecompressBuffer.reset(keyTempBuffer.getData(), compressedKeyLen); CompressionInputStream deflatFilter = codec.createInputStream( keyDecompressBuffer, keyDecompressor); keyDecompressedData.reset(); keyDecompressedData.write(compressedIn, currentKeyLength); keyDataIn.reset(keyDecompressedData.getData(), currentKeyLength); currentKey.readFields(keyDataIn); } else { colValLenBufferReadIn[i].reset(currentKey.allCellValLenBuffer[i] .getData(), currentKey.allCellValLenBuffer[i].getLength()); columnRowReadIndex[i] = 0;