public static HFileDataBlockEncoder createFromFileInfo( FileInfo fileInfo) throws IOException { DataBlockEncoding encoding = DataBlockEncoding.NONE; byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING); if (dataBlockEncodingType != null) { String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType); try { encoding = DataBlockEncoding.valueOf(dataBlockEncodingStr); } catch (IllegalArgumentException ex) { throw new IOException("Invalid data block encoding type in file info: " + dataBlockEncodingStr, ex); } } if (encoding == DataBlockEncoding.NONE) { return NoOpDataBlockEncoder.INSTANCE; } return new HFileDataBlockEncoderImpl(encoding); }
@Override public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) { if (!useEncodedScanner(isCompaction)) { return DataBlockEncoding.NONE; } return encoding; }
@Override public HFileBlock diskToCacheFormat(HFileBlock block, boolean isCompaction) { if (block.getBlockType() == BlockType.DATA) { if (!useEncodedScanner(isCompaction)) { // Unencoded block, and we don't want to encode in cache. return block; } // Encode the unencoded block with the in-cache encoding. return encodeDataBlock(block, inCache, block.doesIncludeMemstoreTS()); } if (block.getBlockType() == BlockType.ENCODED_DATA) { if (block.getDataBlockEncodingId() == onDisk.getId()) { // The block is already in the desired in-cache encoding. return block; } // We don't want to re-encode a block in a different encoding. The HFile // reader should have been instantiated in such a way that we would not // have to do this. throw new AssertionError("Expected on-disk data block encoding " + onDisk + ", got " + block.getDataBlockEncoding()); } return block; }
dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo); fsBlockReader.setDataBlockEncoder(dataBlockEncoder);
/** * Precondition: a non-encoded buffer. * Postcondition: on-disk encoding. */ @Override public Pair<ByteBuffer, BlockType> beforeWriteToDisk(ByteBuffer in, boolean includesMemstoreTS, byte[] dummyHeader) { if (onDisk == DataBlockEncoding.NONE) { // there is no need to encode the block before writing it to disk return new Pair<ByteBuffer, BlockType>(in, BlockType.DATA); } ByteBuffer encodedBuffer = encodeBufferToHFileBlockBuffer(in, onDisk, includesMemstoreTS, dummyHeader); return new Pair<ByteBuffer, BlockType>(encodedBuffer, BlockType.ENCODED_DATA); }
dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo, preferredEncodingInCache); fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
private HFileBlock encodeDataBlock(HFileBlock block, DataBlockEncoding algo, boolean includesMemstoreTS) { ByteBuffer compressedBuffer = encodeBufferToHFileBlockBuffer( block.getBufferWithoutHeader(), algo, includesMemstoreTS, block.getDummyHeaderForVersion()); int sizeWithoutHeader = compressedBuffer.limit() - block.headerSize(); HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA, block.getOnDiskSizeWithoutHeader(), sizeWithoutHeader, block.getPrevBlockOffset(), compressedBuffer, HFileBlock.FILL_HEADER, block.getOffset(), includesMemstoreTS, block.getMinorVersion(), block.getBytesPerChecksum(), block.getChecksumType(), block.getOnDiskDataSizeWithHeader()); block.passSchemaMetricsTo(encodedBlock); return encodedBlock; }
public HFileWriterImpl(final Configuration conf, CacheConfig cacheConf, Path path, FSDataOutputStream outputStream, CellComparator comparator, HFileContext fileContext) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.hFileContext = fileContext; DataBlockEncoding encoding = hFileContext.getDataBlockEncoding(); if (encoding != DataBlockEncoding.NONE) { this.blockEncoder = new HFileDataBlockEncoderImpl(encoding); } else { this.blockEncoder = NoOpDataBlockEncoder.INSTANCE; } this.comparator = comparator != null ? comparator : CellComparator.getInstance(); closeOutputStream = path != null; this.cacheConf = cacheConf; float encodeBlockSizeRatio = conf.getFloat(UNIFIED_ENCODED_BLOCKSIZE_RATIO, 1f); this.encodedBlockSizeLimit = (int)(hFileContext.getBlocksize() * encodeBlockSizeRatio); finishInit(conf); if (LOG.isTraceEnabled()) { LOG.trace("Writer" + (path != null ? " for " + path : "") + " initialized with cacheConf: " + cacheConf + " comparator: " + comparator.getClass().getSimpleName() + " fileContext: " + fileContext); } }
@Override public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) { if (!useEncodedScanner(isCompaction)) { return DataBlockEncoding.NONE; } return inCache; }
dataBlockEncoder = HFileDataBlockEncoderImpl.createFromFileInfo(fileInfo); fsBlockReaderV2.setDataBlockEncoder(dataBlockEncoder);
/** * @return All possible data block encoding configurations */ @Parameters public static Collection<Object[]> getAllConfigurations() { List<Object[]> configurations = new ArrayList<>(); for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) { for (boolean includesMemstoreTS : new boolean[] { false, true }) { HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ? NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo); configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) }); } } return configurations; } }
@Override public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) { if (!useEncodedScanner(isCompaction)) { return DataBlockEncoding.NONE; } return encoding; }
public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly) throws Exception { Map<HStore, HFileDataBlockEncoder> replaceBlockCache = new HashMap<>(); for (HStore store : r.getStores()) { HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder(); replaceBlockCache.put(store, blockEncoder); final DataBlockEncoding inCache = DataBlockEncoding.PREFIX; final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE : inCache; ((HStore)store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk)); } majorCompaction(); // restore settings for (Entry<HStore, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) { ((HStore)entry.getKey()).setDataBlockEncoderInTest(entry.getValue()); } }
this.dataBlockEncoder = new HFileDataBlockEncoderImpl(family.getDataBlockEncoding());
DataBlockEncoding.FAST_DIFF; HFileDataBlockEncoder dataBlockEncoder = new HFileDataBlockEncoderImpl( dataBlockEncoderAlgo); cacheConf = new CacheConfig(conf);
FSDataOutputStream os = fs.create(path); HFileDataBlockEncoder dataBlockEncoder = (encoding != DataBlockEncoding.NONE) ? new HFileDataBlockEncoderImpl(encoding) : NoOpDataBlockEncoder.INSTANCE; HFileContext meta = new HFileContextBuilder() .withCompression(algo)
public AbstractHFileWriter(CacheConfig cacheConf, FSDataOutputStream outputStream, Path path, KVComparator comparator, HFileContext fileContext) { this.outputStream = outputStream; this.path = path; this.name = path != null ? path.getName() : outputStream.toString(); this.hFileContext = fileContext; DataBlockEncoding encoding = hFileContext.getDataBlockEncoding(); if (encoding != DataBlockEncoding.NONE) { this.blockEncoder = new HFileDataBlockEncoderImpl(encoding); } else { this.blockEncoder = NoOpDataBlockEncoder.INSTANCE; } this.comparator = comparator != null ? comparator : KeyValue.COMPARATOR; closeOutputStream = path != null; this.cacheConf = cacheConf; }
public static HFileDataBlockEncoder createFromFileInfo( FileInfo fileInfo) throws IOException { DataBlockEncoding encoding = DataBlockEncoding.NONE; byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING); if (dataBlockEncodingType != null) { String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType); try { encoding = DataBlockEncoding.valueOf(dataBlockEncodingStr); } catch (IllegalArgumentException ex) { throw new IOException("Invalid data block encoding type in file info: " + dataBlockEncodingStr, ex); } } if (encoding == DataBlockEncoding.NONE) { return NoOpDataBlockEncoder.INSTANCE; } return new HFileDataBlockEncoderImpl(encoding); }
return new HFileDataBlockEncoderImpl(onDisk, inCache);
/** * @return All possible data block encoding configurations */ @Parameters public static Collection<Object[]> getAllConfigurations() { List<Object[]> configurations = new ArrayList<>(); for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) { for (boolean includesMemstoreTS : new boolean[] { false, true }) { HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ? NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo); configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) }); } } return configurations; } }