@Override public CompletableFuture<Long> flush(boolean sync) { CompletableFuture<Long> future = new CompletableFuture<>(); ByteArrayOutputStream buffer = this.buffer; this.buffer = new ByteArrayOutputStream(); executor.execute(() -> flush0(future, buffer, sync)); return future; }
if (compressionAlgorithm != NONE) { compressor = compressionAlgorithm.getCompressor(); compressedByteStream = new ByteArrayOutputStream(); try { compressionStream = cryptoByteStream = new ByteArrayOutputStream(); iv = new byte[cryptoContext.getCipher().getIvLength()]; new SecureRandom().nextBytes(iv);
ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream out = new DataOutputStream(baos); for (Cell cell : kvs) {
/** * @param dataBlockEncoder data block encoding algorithm to use */ public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) { if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) { throw new RuntimeException("Unsupported value of bytesPerChecksum. " + " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " + fileContext.getBytesPerChecksum()); } this.dataBlockEncoder = dataBlockEncoder != null? dataBlockEncoder: NoOpDataBlockEncoder.INSTANCE; this.dataBlockEncodingCtx = this.dataBlockEncoder. newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); // TODO: This should be lazily instantiated since we usually do NOT need this default encoder this.defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); // TODO: Set BAOS initial size. Use fileContext.getBlocksize() and add for header/checksum baosInMemory = new ByteArrayOutputStream(); prevOffsetByType = new long[BlockType.values().length]; for (int i = 0; i < prevOffsetByType.length; ++i) { prevOffsetByType[i] = UNSET; } // TODO: Why fileContext saved away when we have dataBlockEncoder and/or // defaultDataBlockEncoder? this.fileContext = fileContext; }
BlockingRpcConnection(BlockingRpcClient rpcClient, ConnectionId remoteId) throws IOException { super(rpcClient.conf, AbstractRpcClient.WHEEL_TIMER, remoteId, rpcClient.clusterId, rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.codec, rpcClient.compressor); this.rpcClient = rpcClient; if (remoteId.getAddress().isUnresolved()) { throw new UnknownHostException("unknown host: " + remoteId.getAddress().getHostName()); } this.connectionHeaderPreamble = getConnectionHeaderPreamble(); ConnectionHeader header = getConnectionHeader(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4 + header.getSerializedSize()); DataOutputStream dos = new DataOutputStream(baos); dos.writeInt(header.getSerializedSize()); header.writeTo(dos); assert baos.size() == 4 + header.getSerializedSize(); this.connectionHeaderWithLength = baos.getBuffer(); UserGroupInformation ticket = remoteId.ticket.getUGI(); this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to " + remoteId.getAddress().toString() + ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName())); if (this.rpcClient.conf.getBoolean(BlockingRpcClient.SPECIFIC_WRITE_THREAD, false)) { callSender = new CallSender(threadName, this.rpcClient.conf); callSender.start(); } else { callSender = null; } }
@Test public void testWriteTag() throws IOException { byte[] tags = Bytes.toBytes("---tags---"); int tagOffset = 3; int length = 4; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, Bytes.toBytes("value"), 0, 5, tags, tagOffset, length); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } }
@Test public void testWriteValue() throws IOException { byte[] value = Bytes.toBytes("---value---"); int valueOffset = 3; int valueLength = 5; IndividualBytesFieldCell cell = new IndividualBytesFieldCell(Bytes.toBytes("row"), 0, 3, Bytes.toBytes("family"), 0, 6, Bytes.toBytes("qualifier"), 0, 9, 0L, KeyValue.Type.Put, 0, value, valueOffset, valueLength, Bytes.toBytes("value"), 0, 5); try (ByteArrayOutputStream output = new ByteArrayOutputStream(300)) { cell.write(output, true); byte[] buf = output.toByteArray(); assertEquals(cell.getSerializedSize(true), buf.length); } } }
@Test public void testLegacySerialization() throws IOException { ByteArrayOutputStream data = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(data); output.writeLong(100); output.writeLong(200); TimeRangeTracker tgt = TimeRangeTracker.parseFrom(data.toByteArray()); assertEquals(100, tgt.getMin()); assertEquals(200, tgt.getMax()); }
static ByteBuffer encodeKeyValues(DataBlockEncoding encoding, List<KeyValue> kvs, HFileBlockEncodingContext encodingContext, boolean useOffheapData) throws IOException { DataBlockEncoder encoder = encoding.getEncoder(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvs) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = new byte[baos.size() - ENCODED_DATA_OFFSET]; System.arraycopy(baos.toByteArray(), ENCODED_DATA_OFFSET, encodedData, 0, encodedData.length); if (useOffheapData) { ByteBuffer bb = ByteBuffer.allocateDirect(encodedData.length); bb.put(encodedData); bb.rewind(); return bb; } return ByteBuffer.wrap(encodedData); }
private void writeBlock(List<Cell> kvs, HFileContext fileContext, boolean useTags) throws IOException { HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HConstants.HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (Cell kv : kvs) { blockEncoder.encode(kv, context, dos); } }
/** * @param ic An instance of IndividualBytesFieldCell to compare. * @param kv An instance of KeyValue to compare. * @param withTags Whether to write tags. */ private void testWriteIntoOutputStream(IndividualBytesFieldCell ic, KeyValue kv, boolean withTags) throws IOException { ByteArrayOutputStream outIC = new ByteArrayOutputStream(ic.getSerializedSize(withTags)); ByteArrayOutputStream outKV = new ByteArrayOutputStream(kv.getSerializedSize(withTags)); // compare the number of bytes written assertEquals(kv.write(outKV, withTags), ic.write(outIC, withTags)); // compare the underlying byte array assertArrayEquals(outKV.getBuffer(), outIC.getBuffer()); }
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags) throws IOException { int size; HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext( blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER, block.getHFileContext()); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(block.getDummyHeaderForVersion()); DataOutputStream dos = new DataOutputStream(baos); blockEncoder.startBlockEncoding(context, dos); for (KeyValue kv : kvs) { blockEncoder.encode(kv, context, dos); } blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA); byte[] encodedBytes = baos.toByteArray(); size = encodedBytes.length - block.getDummyHeaderForVersion().length; return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes), HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1, block.getHFileContext()); }
private void testEncodersOnDataset(List<KeyValue> kvList, boolean includesMemstoreTS, boolean includesTags) throws IOException { ByteBuffer unencodedDataBuf = RedundantKVGenerator.convertKvToByteBuffer(kvList, includesMemstoreTS); HFileContext fileContext = new HFileContextBuilder().withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTags).build(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { DataBlockEncoder encoder = encoding.getEncoder(); if (encoder == null) { continue; } HFileBlockEncodingContext encodingContext = new HFileBlockDefaultEncodingContext(encoding, HFILEBLOCK_DUMMY_HEADER, fileContext); ByteArrayOutputStream baos = new ByteArrayOutputStream(); baos.write(HFILEBLOCK_DUMMY_HEADER); DataOutputStream dos = new DataOutputStream(baos); encoder.startBlockEncoding(encodingContext, dos); for (KeyValue kv : kvList) { encoder.encode(kv, encodingContext, dos); } encoder.endBlockEncoding(encodingContext, dos, baos.getBuffer()); byte[] encodedData = baos.toByteArray(); testAlgorithm(encodedData, unencodedDataBuf, encoder); } }
@Test public void testCompressUncompressTags1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); ByteBuffer ib = ByteBuffer.wrap(kv1.getTagsArray()); context.compressTags(baos, ib, kv1.getTagsOffset(), tagsLength1); KeyValue kv2 = createKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); ib = ByteBuffer.wrap(kv2.getTagsArray()); context.compressTags(baos, ib, kv2.getTagsOffset(), tagsLength2); context.clear(); byte[] dest = new byte[tagsLength1]; ByteBuffer ob = ByteBuffer.wrap(baos.toByteArray()); context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }
onDiskBlockBytesWithHeader = new ByteArrayOutputStream(compressAndEncryptDat.getLength());
@Test public void testCompressUncompressTags2() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); KeyValue kv1 = createKVWithTags(1); int tagsLength1 = kv1.getTagsLength(); context.compressTags(baos, kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1); KeyValue kv2 = createKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); context.compressTags(baos, kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2); context.clear(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.getBuffer()); byte[] dest = new byte[tagsLength1]; context.uncompressTags(bais, dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(bais, dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }
@Test public void testCompressUncompressTagsWithOffheapKeyValue2() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream daos = new ByteBufferWriterDataOutputStream(baos); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); ByteBufferExtendedCell kv1 = (ByteBufferExtendedCell)createOffheapKVWithTags(1); int tagsLength1 = kv1.getTagsLength(); context.compressTags(daos, kv1.getTagsByteBuffer(), kv1.getTagsPosition(), tagsLength1); ByteBufferExtendedCell kv2 = (ByteBufferExtendedCell)createOffheapKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); context.compressTags(daos, kv2.getTagsByteBuffer(), kv2.getTagsPosition(), tagsLength2); context.clear(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.getBuffer()); byte[] dest = new byte[tagsLength1]; context.uncompressTags(bais, dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(bais, dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }
@Test public void testCompressUncompressTagsWithOffheapKeyValue1() throws Exception { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream daos = new ByteBufferWriterDataOutputStream(baos); TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE); ByteBufferExtendedCell kv1 = (ByteBufferExtendedCell)createOffheapKVWithTags(2); int tagsLength1 = kv1.getTagsLength(); context.compressTags(daos, kv1.getTagsByteBuffer(), kv1.getTagsPosition(), tagsLength1); ByteBufferExtendedCell kv2 = (ByteBufferExtendedCell)createOffheapKVWithTags(3); int tagsLength2 = kv2.getTagsLength(); context.compressTags(daos, kv2.getTagsByteBuffer(), kv2.getTagsPosition(), tagsLength2); context.clear(); byte[] dest = new byte[tagsLength1]; ByteBuffer ob = ByteBuffer.wrap(baos.getBuffer()); context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength1); assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0, tagsLength1)); dest = new byte[tagsLength2]; context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength2); assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0, tagsLength2)); }
BlockingRpcConnection(BlockingRpcClient rpcClient, ConnectionId remoteId) throws IOException { super(rpcClient.conf, AbstractRpcClient.WHEEL_TIMER, remoteId, rpcClient.clusterId, rpcClient.userProvider.isHBaseSecurityEnabled(), rpcClient.codec, rpcClient.compressor); this.rpcClient = rpcClient; if (remoteId.getAddress().isUnresolved()) { throw new UnknownHostException("unknown host: " + remoteId.getAddress().getHostName()); } this.connectionHeaderPreamble = getConnectionHeaderPreamble(); ConnectionHeader header = getConnectionHeader(); ByteArrayOutputStream baos = new ByteArrayOutputStream(4 + header.getSerializedSize()); DataOutputStream dos = new DataOutputStream(baos); dos.writeInt(header.getSerializedSize()); header.writeTo(dos); assert baos.size() == 4 + header.getSerializedSize(); this.connectionHeaderWithLength = baos.getBuffer(); UserGroupInformation ticket = remoteId.ticket.getUGI(); this.threadName = "IPC Client (" + this.rpcClient.socketFactory.hashCode() + ") connection to " + remoteId.getAddress().toString() + ((ticket == null) ? " from an unknown user" : (" from " + ticket.getUserName())); if (this.rpcClient.conf.getBoolean(BlockingRpcClient.SPECIFIC_WRITE_THREAD, false)) { callSender = new CallSender(threadName, this.rpcClient.conf); callSender.start(); } else { callSender = null; } }
@Test public void testLegacySerialization() throws IOException { ByteArrayOutputStream data = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(data); output.writeLong(100); output.writeLong(200); TimeRangeTracker tgt = TimeRangeTracker.parseFrom(data.toByteArray()); assertEquals(100, tgt.getMin()); assertEquals(200, tgt.getMax()); }