static ByteBuffer encodeKeyValues(DataBlockEncoding encoding, List<KeyValue> kvs, HFileBlockEncodingContext encodingContext, boolean useOffheapData) throws IOException { DataBlockEncoder encoder = encoding.getEncoder(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvs) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = new byte[baos.size() - ENCODED_DATA_OFFSET]; System.arraycopy(baos.toByteArray(), ENCODED_DATA_OFFSET, encodedData, 0, encodedData.length); if (useOffheapData) { ByteBuffer bb = ByteBuffer.allocateDirect(encodedData.length); bb.put(encodedData); bb.rewind(); return bb; } return ByteBuffer.wrap(encodedData); }
cryptoByteStream.reset(); cryptoByteStream.write(headerBytes); compressedByteStream.reset(); compressionStream.resetState(); compressionStream.write(uncompressedBytesWithHeaderBuffer, compressionStream.flush(); compressionStream.finish(); byte[] plaintext = compressedByteStream.toByteArray(); plaintextLength = plaintext.length; in = new ByteArrayInputStream(plaintext); cryptoByteStream.write(ivLength); if (ivLength > 0) { encryptor.setIv(iv); cryptoByteStream.write(iv); Encryption.incrementIv(iv, 1 + (cryptoByteStream.size() / encryptor.getBlockSize())); return new Bytes(cryptoByteStream.getBuffer(), 0, cryptoByteStream.size()); } else { cryptoByteStream.write(0); return new Bytes(cryptoByteStream.getBuffer(), 0, cryptoByteStream.size()); compressedByteStream.reset(); compressedByteStream.write(headerBytes); compressionStream.resetState();
@Override public void writeInt(int i) { buffer.writeInt(i); }
/** * Returns the header or the compressed data (or uncompressed data when not * using compression) as a byte array. Can be called in the "writing" state * or in the "block ready" state. If called in the "writing" state, * transitions the writer to the "block ready" state. This returns * the header + data + checksums stored on disk. * * @return header and data as they would be stored on disk in a byte array * @throws IOException */ byte[] getHeaderAndDataForTest() throws IOException { ensureBlockReady(); // This is not very optimal, because we are doing an extra copy. // But this method is used only by unit tests. byte[] output = new byte[onDiskBlockBytesWithHeader.size() + onDiskChecksum.length]; System.arraycopy(onDiskBlockBytesWithHeader.getBuffer(), 0, output, 0, onDiskBlockBytesWithHeader.size()); System.arraycopy(onDiskChecksum, 0, output, onDiskBlockBytesWithHeader.size(), onDiskChecksum.length); return output; }
/** * Clones the header followed by the uncompressed data, even if using * compression. This is needed for storing uncompressed blocks in the block * cache. Can be called in the "writing" state or the "block ready" state. * Returns only the header and data, does not include checksum data. * * @return Returns a copy of uncompressed block bytes for caching on write */ @VisibleForTesting ByteBuffer cloneUncompressedBufferWithHeader() { expectState(State.BLOCK_READY); byte[] uncompressedBlockBytesWithHeader = baosInMemory.toByteArray(); int numBytes = (int) ChecksumUtil.numBytes( onDiskBlockBytesWithHeader.size(), fileContext.getBytesPerChecksum()); putHeader(uncompressedBlockBytesWithHeader, 0, onDiskBlockBytesWithHeader.size() + numBytes, baosInMemory.size(), onDiskBlockBytesWithHeader.size()); return ByteBuffer.wrap(uncompressedBlockBytesWithHeader); }
if (blockType == BlockType.DATA) { this.dataBlockEncoder.endBlockEncoding(dataBlockEncodingCtx, userDataStream, baosInMemory.getBuffer(), blockType); blockType = dataBlockEncodingCtx.getBlockType(); if (blockType == BlockType.DATA || blockType == BlockType.ENCODED_DATA) { compressAndEncryptDat = dataBlockEncodingCtx. compressAndEncrypt(baosInMemory.getBuffer(), 0, baosInMemory.size()); } else { compressAndEncryptDat = defaultBlockEncodingCtx. compressAndEncrypt(baosInMemory.getBuffer(), 0, baosInMemory.size()); compressAndEncryptDat = new Bytes(baosInMemory.getBuffer(), 0, baosInMemory.size()); onDiskBlockBytesWithHeader = new ByteArrayOutputStream(compressAndEncryptDat.getLength()); onDiskBlockBytesWithHeader.reset(); onDiskBlockBytesWithHeader.write(compressAndEncryptDat.get(), compressAndEncryptDat.getOffset(), compressAndEncryptDat.getLength()); onDiskBlockBytesWithHeader.size(), fileContext.getBytesPerChecksum()); onDiskBlockBytesWithHeader.size() + numBytes, baosInMemory.size(), onDiskBlockBytesWithHeader.size()); if (onDiskChecksum.length != numBytes) { onDiskChecksum = new byte[numBytes]; onDiskBlockBytesWithHeader.getBuffer(), 0,onDiskBlockBytesWithHeader.size(),
/** * @param ic An instance of IndividualBytesFieldCell to compare. * @param kv An instance of KeyValue to compare. * @param withTags Whether to write tags. */ private void testWriteIntoOutputStream(IndividualBytesFieldCell ic, KeyValue kv, boolean withTags) throws IOException { ByteArrayOutputStream outIC = new ByteArrayOutputStream(ic.getSerializedSize(withTags)); ByteArrayOutputStream outKV = new ByteArrayOutputStream(kv.getSerializedSize(withTags)); // compare the number of bytes written assertEquals(kv.write(outKV, withTags), ic.write(outIC, withTags)); // compare the underlying byte array assertArrayEquals(outKV.getBuffer(), outIC.getBuffer()); }
BlockingRpcConnection(BlockingRpcClient rpcClient, ConnectionId remoteId) throws IOException { super(rpcClient.conf, AbstractRpcClient.WHEEL_TIMER, remoteId, rpcClient.clusterId, rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.codec, rpcClient.compressor); this.rpcClient = rpcClient; if (remoteId.getAddress().isUnresolved()) { throw new UnknownHostException("unknown host: " + remoteId.getAddress().getHostName()); } this.connectionHeaderPreamble = getConnectionHeaderPreamble(); ConnectionHeader header = getConnectionHeader(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4 + header.getSerializedSize()); DataOutputStream dos = new DataOutputStream(baos); dos.writeInt(header.getSerializedSize()); header.writeTo(dos); assert baos.size() == 4 + header.getSerializedSize(); this.connectionHeaderWithLength = baos.getBuffer(); UserGroupInformation ticket = remoteId.ticket.getUGI(); this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to " + remoteId.getAddress().toString() + ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName())); if (this.rpcClient.conf.getBoolean(BlockingRpcClient.SPECIFIC_WRITE_THREAD, false)) { callSender = new CallSender(threadName, this.rpcClient.conf); callSender.start(); } else { callSender = null; } }
@Test public void testWriteTag() throws IOException { byte[] tags = Bytes.toBytes("---tags---"); int tagOffset = 3; int length = 4; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, Bytes.toBytes("value"), 0, 5, tags, tagOffset, length); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } }
@Override public CompletableFuture<Long> flush(boolean sync) { CompletableFuture<Long> future = new CompletableFuture<>(); ByteArrayOutputStream buffer = this.buffer; this.buffer = new ByteArrayOutputStream(); executor.execute(() -> flush0(future, buffer, sync)); return future; }
private void writeBlock(List<Cell> kvs, HFileContext fileContext, boolean useTags) throws IOException { HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (Cell kv : kvs) { blockEncoder.encode(kv, context, dos); } }
@Override public int buffered() { return buffer.size(); }
private void putHeader(ByteArrayOutputStream dest, int onDiskSize, int uncompressedSize, int onDiskDataSize) { putHeader(dest.getBuffer(),0, onDiskSize, uncompressedSize, onDiskDataSize); }
/** * Starts writing into the block. The previous block's data is discarded. * * @return the stream the user can write their data into * @throws IOException */ DataOutputStream startWriting(BlockType newBlockType) throws IOException { if (state == State.BLOCK_READY && startOffset != -1) { // We had a previous block that was written to a stream at a specific // offset. Save that offset as the last offset of a block of that type. prevOffsetByType[blockType.getId()] = startOffset; } startOffset = -1; blockType = newBlockType; baosInMemory.reset(); baosInMemory.write(HConstants.HFILEBLOCK_DUMMY_HEADER); state = State.WRITING; // We will compress it later in finishBlock() userDataStream = new ByteBufferWriterDataOutputStream(baosInMemory); if (newBlockType == BlockType.DATA) { this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, userDataStream); } this.unencodedDataSizeWritten = 0; this.encodedDataSizeWritten = 0; return userDataStream; }
/** * Clones the header followed by the on-disk (compressed/encoded/encrypted) data. This is * needed for storing packed blocks in the block cache. Expects calling semantics identical to * {@link #getUncompressedBufferWithHeader()}. Returns only the header and data, * Does not include checksum data. * * @return Returns a copy of block bytes for caching on write */ private ByteBuffer cloneOnDiskBufferWithHeader() { expectState(State.BLOCK_READY); return ByteBuffer.wrap(onDiskBlockBytesWithHeader.toByteArray()); }
@Override public void write(byte[] b, int off, int len) { buffer.write(b, off, len); }
@Test public void testCompressUncompressTags2() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(1); int tagsLength1 = kv1.getTagsLength(); context.compressTags(baos, kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1); KeyValue kv2 = createKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); context.compressTags(baos, kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2); context.clear(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.getBuffer()); byte[] dest = new byte[tagsLength1]; context.uncompressTags(bais, dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(bais, dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }
ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(baos); for (Cell cell : kvs) { return ByteBuffer.wrap(baos.getBuffer(), 0, baos.size());
@Test public void testLegacySerialization() throws IOException { ByteArrayOutputStream data = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(data); output.writeLong(100); output.writeLong(200); TimeRangeTracker tgt = TimeRangeTracker.parseFrom(data.toByteArray()); assertEquals(100, tgt.getMin()); assertEquals(200, tgt.getMax()); }
public void flush() throws IOException { int onDiskDataSize = 0; if (startOffset >= 0) { onDiskDataSize = out.size() - startOffset; } out.writeInt(rowsOffsetBAOS.size() / 4); if (rowsOffsetBAOS.size() > 0) { out.write(rowsOffsetBAOS.getBuffer(), 0, rowsOffsetBAOS.size()); } out.writeInt(onDiskDataSize); if (LOG.isTraceEnabled()) { LOG.trace("RowNumber: " + rowsOffsetBAOS.size() / 4 + ", onDiskDataSize: " + onDiskDataSize + ", totalOnDiskSize: " + (out.size() - startOffset)); } }