DataOutputStream dos = hbw.startWriting(bt); int size = rand.nextInt(500); for (int j = 0; j < size; ++j) {
blockWriter.startWriting(BlockType.ROOT_INDEX); rootChunk.writeRoot(blockStream); if (midKeyMetadata != null)
hbw.startWriting(BlockType.DATA).write(String.valueOf(rand.nextInt(1000)).getBytes()); long blockOffset = outputStream.getPos(); hbw.writeHeaderAndData(outputStream);
/** * Ready a new block for writing. * * @throws IOException */ protected void newBlock() throws IOException { // This is where the next block begins. fsBlockWriter.startWriting(BlockType.DATA); firstCellInBlock = null; if (lastCell != null) { lastCellOfPreviousBlock = lastCell; } }
/** * Ready a new block for writing. * * @throws IOException */ private void newBlock() throws IOException { // This is where the next block begins. fsBlockWriter.startWriting(BlockType.DATA); firstKeyInBlock = null; }
private void writeInlineBlocks(HFileBlock.Writer hbw, FSDataOutputStream outputStream, HFileBlockIndex.BlockIndexWriter biw, boolean isClosing) throws IOException { while (biw.shouldWriteBlock(isClosing)) { long offset = outputStream.getPos(); biw.writeInlineBlock(hbw.startWriting(biw.getInlineBlockType())); hbw.writeHeaderAndData(outputStream); biw.blockWritten(offset, hbw.getOnDiskSizeWithHeader(), hbw.getUncompressedSizeWithoutHeader()); LOG.info("Wrote an inline index block at " + offset + ", size " + hbw.getOnDiskSizeWithHeader()); } }
final List<ByteBuffer> encodedBlocks = new ArrayList<>(); for (int blockId = 0; blockId < numBlocks; ++blockId) { hbw.startWriting(BlockType.DATA); writeTestKeyValues(hbw, blockId, includesMemstoreTS, includesTag); hbw.writeHeaderAndData(os);
long totalSize = 0; for (int blockId = 0; blockId < 2; ++blockId) { DataOutputStream dos = hbw.startWriting(BlockType.DATA); for (int i = 0; i < 1234; ++i) dos.writeInt(i);
long totalSize = 0; for (int blockId = 0; blockId < 2; ++blockId) { DataOutputStream dos = hbw.startWriting(BlockType.DATA); for (int i = 0; i < 1234; ++i) dos.writeInt(i);
DataOutputStream dos = hbw.startWriting(BlockType.DATA); for (; dataSize < 6 * bytesPerChecksum;) { for (int i = 0; i < 1234; ++i) {
DataOutputStream dos = blockWriter.startWriting(BlockType.META); metaData.get(i).write(dos); metaBlockIndexWriter.writeSingleLevelIndex(blockWriter.startWriting( BlockType.ROOT_INDEX), "meta"); blockWriter.writeHeaderAndData(outputStream); writeFileInfo(trailer, blockWriter.startWriting(BlockType.FILE_INFO)); blockWriter.writeHeaderAndData(outputStream); totalUncompressedBytes += blockWriter.getUncompressedSizeWithHeader();
private void writeIntermediateBlock(FSDataOutputStream out, BlockIndexChunk parent, BlockIndexChunk curChunk) throws IOException { long beginOffset = out.getPos(); DataOutputStream dos = blockWriter.startWriting( BlockType.INTERMEDIATE_INDEX); curChunk.writeNonRoot(dos);
.build(); HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta); DataOutputStream dos = hbw.startWriting(BlockType.DATA); for (int i = 0; i < 1000; ++i) { dos.writeInt(i);
@Test public void testNewBlocksHaveDefaultChecksum() throws IOException { Path path = new Path(TEST_UTIL.getDataTestDir(), "default_checksum"); FSDataOutputStream os = fs.create(path); HFileContext meta = new HFileContextBuilder().build(); HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta); DataOutputStream dos = hbw.startWriting(BlockType.DATA); for (int i = 0; i < 1000; ++i) dos.writeInt(i); hbw.writeHeaderAndData(os); int totalSize = hbw.getOnDiskSizeWithHeader(); os.close(); // Use hbase checksums. assertEquals(true, hfs.useHBaseChecksum()); FSDataInputStreamWrapper is = new FSDataInputStreamWrapper(fs, path); meta = new HFileContextBuilder().withHBaseCheckSum(true).build(); HFileBlock.FSReader hbr = new HFileBlock.FSReaderImpl( is, totalSize, (HFileSystem) fs, path, meta); HFileBlock b = hbr.readBlockData(0, -1, false, false); assertEquals(b.getChecksumType(), ChecksumType.getDefaultChecksumType().getCode()); }
/** Gives inline block writers an opportunity to contribute blocks. */ private void writeInlineBlocks(boolean closing) throws IOException { for (InlineBlockWriter ibw : inlineBlockWriters) { while (ibw.shouldWriteBlock(closing)) { long offset = outputStream.getPos(); boolean cacheThisBlock = ibw.getCacheOnWrite(); ibw.writeInlineBlock(blockWriter.startWriting( ibw.getInlineBlockType())); blockWriter.writeHeaderAndData(outputStream); ibw.blockWritten(offset, blockWriter.getOnDiskSizeWithHeader(), blockWriter.getUncompressedSizeWithoutHeader()); totalUncompressedBytes += blockWriter.getUncompressedSizeWithHeader(); if (cacheThisBlock) { doCacheOnWrite(offset); } } } }
static HFileBlock.Writer createTestV2Block(Compression.Algorithm algo, boolean includesMemstoreTS, boolean includesTag) throws IOException { final BlockType blockType = BlockType.DATA; HFileContext meta = new HFileContextBuilder() .withCompression(algo) .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTag) .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM) .build(); HFileBlock.Writer hbw = new HFileBlock.Writer(null, meta); DataOutputStream dos = hbw.startWriting(blockType); writeTestBlockContents(dos); dos.flush(); hbw.ensureBlockReady(); assertEquals(1000 * 4, hbw.getUncompressedSizeWithoutHeader()); hbw.release(); return hbw; }
/** * Takes the given {@link BlockWritable} instance, creates a new block of * its appropriate type, writes the writable into this block, and flushes * the block into the output stream. The writer is instructed not to buffer * uncompressed bytes for cache-on-write. * * @param bw the block-writable object to write as a block * @param out the file system output stream * @throws IOException */ void writeBlock(BlockWritable bw, FSDataOutputStream out) throws IOException { bw.writeToBlock(startWriting(bw.getBlockType())); writeHeaderAndData(out); }
private int writeBlock(FSDataOutputStream os, HFileContext fileContext, int size) throws IOException { HFileBlock.Writer hbw = new HFileBlock.Writer(null, fileContext); DataOutputStream dos = hbw.startWriting(BlockType.DATA); for (int j = 0; j < size; j++) { dos.writeInt(j); } hbw.writeHeaderAndData(os); LOG.info("Wrote a block at " + os.getPos() + " with" + " onDiskSizeWithHeader=" + hbw.getOnDiskSizeWithHeader() + " uncompressedSizeWithoutHeader=" + hbw.getOnDiskSizeWithoutHeader() + " uncompressedSizeWithoutHeader=" + hbw.getUncompressedSizeWithoutHeader()); return hbw.getOnDiskSizeWithHeader(); }
/** * Ready a new block for writing. * * @throws IOException */ protected void newBlock() throws IOException { // This is where the next block begins. blockWriter.startWriting(BlockType.DATA); firstCellInBlock = null; if (lastCell != null) { lastCellOfPreviousBlock = lastCell; } }
private void writeInlineBlocks(HFileBlock.Writer hbw, FSDataOutputStream outputStream, HFileBlockIndex.BlockIndexWriter biw, boolean isClosing) throws IOException { while (biw.shouldWriteBlock(isClosing)) { long offset = outputStream.getPos(); biw.writeInlineBlock(hbw.startWriting(biw.getInlineBlockType())); hbw.writeHeaderAndData(outputStream); biw.blockWritten(offset, hbw.getOnDiskSizeWithHeader(), hbw.getUncompressedSizeWithoutHeader()); LOG.info("Wrote an inline index block at " + offset + ", size " + hbw.getOnDiskSizeWithHeader()); } }