public static BlockType parse(byte[] buf, int offset, int length) throws IOException { if (length != MAGIC_LENGTH) { throw new IOException("Magic record of invalid length: " + Bytes.toStringBinary(buf, offset, length)); } for (BlockType blockType : values()) if (Bytes.compareTo(blockType.magic, 0, MAGIC_LENGTH, buf, offset, MAGIC_LENGTH) == 0) return blockType; throw new IOException("Invalid HFile block magic: " + Bytes.toStringBinary(buf, offset, MAGIC_LENGTH)); }
/** * Use this instead of {@link #ordinal()}. They work exactly the same, except * DATA and ENCODED_DATA get the same id using this method (overridden for * {@link #ENCODED_DATA}). * @return block type id from 0 to the number of block types - 1 */ public int getId() { // Default implementation, can be overridden for individual enum members. return ordinal(); }
public static BlockType read(DataInputStream in) throws IOException { byte[] buf = new byte[MAGIC_LENGTH]; in.readFully(buf); return parse(buf, 0, buf.length); }
for (int i = 0; i < NUM_TEST_BLOCKS; ++i) { long pos = os.getPos(); int blockTypeOrdinal = rand.nextInt(BlockType.values().length); if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) { blockTypeOrdinal = BlockType.DATA.ordinal(); BlockType bt = BlockType.values()[blockTypeOrdinal]; DataOutputStream dos = hbw.startWriting(bt); int size = rand.nextInt(500);
(HFileBlock) cache.getBlock(cacheKey, cacheBlock, useLock, updateCacheMetrics); if (cachedBlock != null) { if (cacheConf.shouldCacheCompressed(cachedBlock.getBlockType().getCategory())) { HFileBlock compressedBlock = cachedBlock; cachedBlock = compressedBlock.unpack(hfileContext, fsBlockReader); if (cachedBlock.getBlockType().isData() && !actualDataBlockEncoding.equals(expectedDataBlockEncoding)) {
/** * Write the trailer to a data stream. We support writing version 1 for * testing and for determining version 1 trailer size. It is also easy to see * what fields changed in version 2. * * @param outputStream * @throws IOException */ void serialize(DataOutputStream outputStream) throws IOException { HFile.checkFormatVersion(majorVersion); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream baosDos = new DataOutputStream(baos); BlockType.TRAILER.write(baosDos); serializeAsPB(baosDos); // The last 4 bytes of the file encode the major and minor version universally baosDos.writeInt(materializeVersion(majorVersion, minorVersion)); baos.writeTo(outputStream); }
/** * Return true if we may find this type of block in block cache. * <p> * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled} * configuration. */ public boolean shouldReadBlockFromCache(BlockType blockType) { if (cacheDataOnRead) { return true; } if (prefetchOnOpen) { return true; } if (cacheDataOnWrite) { return true; } if (blockType == null) { return true; } if (blockType.getCategory() == BlockCategory.BLOOM || blockType.getCategory() == BlockCategory.INDEX) { return true; } return false; }
/** * Compares the actual type of a block retrieved from cache or disk with its * expected type and throws an exception in case of a mismatch. Expected * block type of {@link BlockType#DATA} is considered to match the actual * block type [@link {@link BlockType#ENCODED_DATA} as well. * @param block a block retrieved from cache or disk * @param expectedBlockType the expected block type, or null to skip the * check */ private void validateBlockType(HFileBlock block, BlockType expectedBlockType) throws IOException { if (expectedBlockType == null) { return; } BlockType actualBlockType = block.getBlockType(); if (expectedBlockType.isData() && actualBlockType.isData()) { // We consider DATA to match ENCODED_DATA for the purpose of this // verification. return; } if (actualBlockType != expectedBlockType) { throw new IOException("Expected block type " + expectedBlockType + ", " + "but got " + actualBlockType + ": " + block); } }
sanityCheckAssertion(BlockType.read(dup), blockType);
/** * Starts writing into the block. The previous block's data is discarded. * * @return the stream the user can write their data into * @throws IOException */ DataOutputStream startWriting(BlockType newBlockType) throws IOException { if (state == State.BLOCK_READY && startOffset != -1) { // We had a previous block that was written to a stream at a specific // offset. Save that offset as the last offset of a block of that type. prevOffsetByType[blockType.getId()] = startOffset; } startOffset = -1; blockType = newBlockType; baosInMemory.reset(); baosInMemory.write(HConstants.HFILEBLOCK_DUMMY_HEADER); state = State.WRITING; // We will compress it later in finishBlock() userDataStream = new ByteBufferWriterDataOutputStream(baosInMemory); if (newBlockType == BlockType.DATA) { this.dataBlockEncoder.startBlockEncoding(dataBlockEncodingCtx, userDataStream); } this.unencodedDataSizeWritten = 0; this.encodedDataSizeWritten = 0; return userDataStream; }
/** * Put the header into the given byte array at the given offset. * @param onDiskSize size of the block on disk header + data + checksum * @param uncompressedSize size of the block after decompression (but * before optional data block decoding) including header * @param onDiskDataSize size of the block on disk with header * and data but not including the checksums */ private void putHeader(byte[] dest, int offset, int onDiskSize, int uncompressedSize, int onDiskDataSize) { offset = blockType.put(dest, offset); offset = Bytes.putInt(dest, offset, onDiskSize - HConstants.HFILEBLOCK_HEADER_SIZE); offset = Bytes.putInt(dest, offset, uncompressedSize - HConstants.HFILEBLOCK_HEADER_SIZE); offset = Bytes.putLong(dest, offset, prevOffset); offset = Bytes.putByte(dest, offset, fileContext.getChecksumType().getCode()); offset = Bytes.putInt(dest, offset, fileContext.getBytesPerChecksum()); Bytes.putInt(dest, offset, onDiskDataSize); }
/** * Deserialize the fixed file trailer from the given stream. The version needs * to already be specified. Make sure this is consistent with * {@link #serialize(DataOutputStream)}. * * @param inputStream * @throws IOException */ void deserialize(DataInputStream inputStream) throws IOException { HFile.checkFormatVersion(majorVersion); BlockType.TRAILER.readAndCheck(inputStream); if (majorVersion > 2 || (majorVersion == 2 && minorVersion >= HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION)) { deserializeFromPB(inputStream); } else { deserializeFromWritable(inputStream); } // The last 4 bytes of the file encode the major and minor version universally int version = inputStream.readInt(); expectMajorVersion(extractMajorVersion(version)); expectMinorVersion(extractMinorVersion(version)); }
for (int i = 0; i < NUM_TEST_BLOCKS; ++i) { long pos = os.getPos(); int blockTypeOrdinal = rand.nextInt(BlockType.values().length); if (blockTypeOrdinal == BlockType.ENCODED_DATA.ordinal()) { blockTypeOrdinal = BlockType.DATA.ordinal(); BlockType bt = BlockType.values()[blockTypeOrdinal]; DataOutputStream dos = hbw.startWriting(bt); int size = rand.nextInt(500);
public byte[] createTestV1Block(Compression.Algorithm algo) throws IOException { Compressor compressor = algo.getCompressor(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); OutputStream os = algo.createCompressionStream(baos, compressor, 0); DataOutputStream dos = new DataOutputStream(os); BlockType.META.write(dos); // Let's make this a meta block. writeTestBlockContents(dos); dos.flush(); algo.returnCompressor(compressor); return baos.toByteArray(); }
/** * If we make sure the block could not be cached, we will not acquire the lock * otherwise we will acquire lock */ public boolean shouldLockOnCacheMiss(BlockType blockType) { if (blockType == null) { return true; } return shouldCacheBlockOnRead(blockType.getCategory()); }
/** * Helper function that updates the local size counter and also updates any * per-cf or per-blocktype metrics it can discern from given * {@link LruCachedBlock} */ private long updateSizeMetrics(LruCachedBlock cb, boolean evict) { long heapsize = cb.heapSize(); BlockType bt = cb.getBuffer().getBlockType(); if (evict) { heapsize *= -1; } if (bt != null && bt.isData()) { dataBlockSize.add(heapsize); } return size.addAndGet(heapsize); }
if (cachedBlock.getBlockType().isData()) { if (updateCacheMetrics) { HFile.DATABLOCK_READ_COUNT.increment(); validateBlockType(hfileBlock, expectedBlockType); HFileBlock unpacked = hfileBlock.unpack(hfileContext, fsBlockReader); BlockType.BlockCategory category = hfileBlock.getBlockType().getCategory(); if (updateCacheMetrics && hfileBlock.getBlockType().isData()) { HFile.DATABLOCK_READ_COUNT.increment();
final int nextBlockOnDiskSize, HFileContext fileContext) throws IOException { buf.rewind(); final BlockType blockType = BlockType.read(buf); final int onDiskSizeWithoutHeader = buf.getInt(Header.ON_DISK_SIZE_WITHOUT_HEADER_INDEX); final int uncompressedSizeWithoutHeader =
prevOffset = prevOffsetByType[blockType.getId()];
private void put20Header(byte[] dest, int offset, int onDiskSize, int uncompressedSize) { offset = blockType.put(dest, offset); offset = Bytes.putInt(dest, offset, onDiskSize - HEADER_SIZE_NO_CHECKSUM); offset = Bytes.putInt(dest, offset, uncompressedSize - HEADER_SIZE_NO_CHECKSUM); Bytes.putLong(dest, offset, prevOffset); } /**