.withCompression(Compression.Algorithm.NONE) .withBytesPerCheckSum(0) .withChecksumType(ChecksumType.NULL) .build(); HFileBlock generated = new HFileBlock(BlockType.DATA,
private HFileBlock getSampleHFileBlock(List<KeyValue> kvs, boolean useTag) { ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); int size = keyValues.limit(); ByteBuffer buf = ByteBuffer.allocate(size + HConstants.HFILEBLOCK_HEADER_SIZE); buf.position(HConstants.HFILEBLOCK_HEADER_SIZE); keyValues.rewind(); buf.put(keyValues); HFileContext meta = new HFileContextBuilder() .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(useTag) .withHBaseCheckSum(true) .withCompression(Algorithm.NONE) .withBlockSize(0) .withChecksumType(ChecksumType.NULL) .build(); HFileBlock b = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, 0, 0, -1, meta); return b; }
HFileContext meta = new HFileContextBuilder() .withBlockSize(BLOCKSIZE_SMALL) .withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES) .build();
BloomType bloomFilterType = familyDescriptor.getBloomFilterType(); HFileContext hFileContext = new HFileContextBuilder().withCompression(compression) .withChecksumType(HStore.getChecksumType(conf)) .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)).withBlockSize(blocksize) .withDataBlockEncoding(familyDescriptor.getDataBlockEncoding()).withIncludesTags(true)
private void testHeaderSizeInCacheWithoutChecksumInternals(boolean useTags) throws IOException { int headerSize = HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM; // Create some KVs and create the block with old-style header. List<KeyValue> kvs = generator.generateTestKeyValues(60, useTags); ByteBuffer keyValues = RedundantKVGenerator.convertKvToByteBuffer(kvs, includesMemstoreTS); int size = keyValues.limit(); ByteBuffer buf = ByteBuffer.allocate(size + headerSize); buf.position(headerSize); keyValues.rewind(); buf.put(keyValues); HFileContext hfileContext = new HFileContextBuilder().withHBaseCheckSum(false) .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(useTags) .withBlockSize(0) .withChecksumType(ChecksumType.NULL) .build(); HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, 0, 0, -1, hfileContext); HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTags); assertEquals(headerSize, cacheBlock.getDummyHeaderForVersion().length); }
.withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES).build();
onDiskDataSizeWithHeader = buf.getInt(Header.ON_DISK_DATA_SIZE_WITH_HEADER_INDEX); fileContextBuilder.withChecksumType(ChecksumType.codeToType(checksumType)); fileContextBuilder.withBytesPerCheckSum(bytesPerChecksum); } else { fileContextBuilder.withChecksumType(ChecksumType.NULL); fileContextBuilder.withBytesPerCheckSum(0);
.withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES).build();
.withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES) .build();
@Test public void testBloomFilter() throws Exception { FileSystem fs = FileSystem.getLocal(conf); conf.setFloat(BloomFilterFactory.IO_STOREFILE_BLOOM_ERROR_RATE, (float) 0.01); conf.setBoolean(BloomFilterFactory.IO_STOREFILE_BLOOM_ENABLED, true); // write the file Path f = new Path(ROOT_DIR, getName()); HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL) .withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES).build(); // Make a store file and write data to it. StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, this.fs) .withFilePath(f) .withBloomType(BloomType.ROW) .withMaxKeyCount(2000) .withFileContext(meta) .build(); bloomWriteRead(writer, fs); }
private HFileContext createFileContext(Compression.Algorithm compression, boolean includeMVCCReadpoint, boolean includesTag, Encryption.Context cryptoContext) { if (compression == null) { compression = HFile.DEFAULT_COMPRESSION_ALGORITHM; } HFileContext hFileContext = new HFileContextBuilder() .withIncludesMvcc(includeMVCCReadpoint) .withIncludesTags(includesTag) .withCompression(compression) .withCompressTags(family.isCompressTags()) .withChecksumType(checksumType) .withBytesPerCheckSum(bytesPerChecksum) .withBlockSize(blocksize) .withHBaseCheckSum(true) .withDataBlockEncoding(family.getDataBlockEncoding()) .withEncryptionContext(cryptoContext) .withCreateTime(EnvironmentEdgeManager.currentTime()) .build(); return hFileContext; }
FSDataOutputStream os = fs.create(path); HFileContext meta = new HFileContextBuilder() .withChecksumType(cktype) .build(); HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta);
"test_cache_on_write"); HFileContext meta = new HFileContextBuilder().withCompression(compress) .withBytesPerCheckSum(CKBYTES).withChecksumType(ChecksumType.NULL) .withBlockSize(DATA_BLOCK_SIZE) .withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding())
cacheConf = new CacheConfig(conf); HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL) .withChecksumType(CKTYPE) .withBytesPerCheckSum(CKBYTES) .withDataBlockEncoding(dataBlockEncoderAlgo)
/** * Test encoding with offheap keyvalue. This test just verifies if the encoders * work with DBB and does not use the getXXXArray() API * @throws IOException */ @Test public void testEncodingWithOffheapKeyValue() throws IOException { // usually we have just block without headers, but don't complicate that try { List<Cell> kvs = generator.generateTestExtendedOffheapKeyValues(60, true); HFileContext meta = new HFileContextBuilder().withIncludesMvcc(includesMemstoreTS) .withIncludesTags(true).withHBaseCheckSum(true).withCompression(Algorithm.NONE) .withBlockSize(0).withChecksumType(ChecksumType.NULL).build(); writeBlock(kvs, meta, true); } catch (IllegalArgumentException e) { fail("No exception should have been thrown"); } }
.withCompression(Algorithm.NONE) .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM) .withChecksumType(ChecksumType.NULL).build(); HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf, HFileBlock.FILL_HEADER, -1, 0, -1, meta);
.withBlockSize(fileContext.getBlocksize()) .withBytesPerCheckSum(0) .withChecksumType(ChecksumType.NULL) // no checksums in cached data .withCompression(fileContext.getCompression()) .withDataBlockEncoding(fileContext.getDataBlockEncoding())
LOG.info("HFile information: " + reader); HFileContext meta = new HFileContextBuilder().withCompression(compress) .withBytesPerCheckSum(CKBYTES).withChecksumType(ChecksumType.NULL) .withBlockSize(DATA_BLOCK_SIZE) .withDataBlockEncoding(NoOpDataBlockEncoder.INSTANCE.getDataBlockEncoding())
.withIncludesMvcc(true).withIncludesTags(true) .withCompressTags(family.isCompressTags()) .withChecksumType(checksumType) .withBytesPerCheckSum(bytesPerChecksum) .withBlockSize(blocksize)
.withCompression(compression) .withCompressTags(family.isCompressTags()) .withChecksumType(HStore.getChecksumType(conf)) .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)) .withBlockSize(family.getBlocksize())