long end = 0; try { end = getTrailer().getLoadOnOpenDataOffset(); if (LOG.isTraceEnabled()) { LOG.trace("Prefetch start " + getPathOffsetEndStr(path, offset, end));
private void readStoreFile(Path storeFilePath) throws Exception { // Open the file reader with block cache disabled. HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf); long offset = 0; while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); offset += block.getOnDiskSizeWithHeader(); } }
private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft) throws IOException { BlockType.TRAILER.write(output); output.writeLong(fft.getFileInfoOffset()); output.writeLong(fft.getLoadOnOpenDataOffset()); output.writeInt(fft.getDataIndexCount()); output.writeLong(fft.getUncompressedDataIndexSize()); output.writeInt(fft.getMetaIndexCount()); output.writeLong(fft.getTotalUncompressedBytes()); output.writeLong(fft.getEntryCount()); output.writeInt(fft.getCompressionCodec().ordinal()); output.writeInt(fft.getNumDataIndexLevels()); output.writeLong(fft.getFirstDataBlockOffset()); output.writeLong(fft.getLastDataBlockOffset()); Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH); output.writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion())); }
/** * Read a storefile in the same manner as a scanner -- using non-positional reads and * without waiting for prefetch to complete. */ private void readStoreFileLikeScanner(Path storeFilePath) throws Exception { // Open the file HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConf, true, conf); do { long offset = 0; while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { HFileBlock block = reader.readBlock(offset, -1, false, /*pread=*/false, false, true, null, null); offset += block.getOnDiskSizeWithHeader(); } } while (!reader.prefetchComplete()); }
throw new IOException("Block index not loaded"); long trailerOffset = trailer.getLoadOnOpenDataOffset(); if (dataBlockOffset < 0 || dataBlockOffset >= trailerOffset) { throw new IOException("Requested block is out of range: " + dataBlockOffset +
private void readStoreFile(Path storeFilePath) throws Exception { // Open the file HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConf, true, conf); while (!reader.prefetchComplete()) { // Sleep for a bit Thread.sleep(1000); } // Check that all of the data blocks were preloaded BlockCache blockCache = cacheConf.getBlockCache().get(); long offset = 0; while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset); boolean isCached = blockCache.getBlock(blockCacheKey, true, false, true) != null; if (block.getBlockType() == BlockType.DATA || block.getBlockType() == BlockType.ROOT_INDEX || block.getBlockType() == BlockType.INTERMEDIATE_INDEX) { assertTrue(isCached); } offset += block.getOnDiskSizeWithHeader(); } }
private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded) throws IOException { assertEquals(version, loaded.getMajorVersion()); assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); assertEquals(Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), loaded.getEntryCount()); if (version == 1) { assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); } if (version == 2) { assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset()); assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels()); assertEquals(expected.createComparator().getClass().getName(), loaded.createComparator().getClass().getName()); assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); assertTrue( expected.createComparator() instanceof CellComparatorImpl); assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); } assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset()); assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes()); }
while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
reader.getTrailer().getLoadOnOpenDataOffset()); HFileBlock block; List<byte[]> blockKeys = new ArrayList<>();
trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize()); while (fsdis.getPos() < trailer.getLoadOnOpenDataOffset()) { LOG.info("Current offset: " + fsdis.getPos() + ", scanning until " + trailer.getLoadOnOpenDataOffset()); HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false) .unpack(context, blockReader);
trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize());
List<Long> cachedBlocksOffset = new ArrayList<>(); Map<Long, HFileBlock> cachedBlocks = new HashMap<>(); while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
private void serializeAsWritable(DataOutputStream output, FixedFileTrailer fft) throws IOException { BlockType.TRAILER.write(output); output.writeLong(fft.getFileInfoOffset()); output.writeLong(fft.getLoadOnOpenDataOffset()); output.writeInt(fft.getDataIndexCount()); output.writeLong(fft.getUncompressedDataIndexSize()); output.writeInt(fft.getMetaIndexCount()); output.writeLong(fft.getTotalUncompressedBytes()); output.writeLong(fft.getEntryCount()); output.writeInt(fft.getCompressionCodec().ordinal()); output.writeInt(fft.getNumDataIndexLevels()); output.writeLong(fft.getFirstDataBlockOffset()); output.writeLong(fft.getLastDataBlockOffset()); Bytes.writeStringFixedSize(output, fft.getComparatorClassName(), MAX_COMPARATOR_NAME_LENGTH); output.writeInt(FixedFileTrailer.materializeVersion(fft.getMajorVersion(), fft.getMinorVersion())); }
private void readStoreFile(Path storeFilePath) throws Exception { // Open the file reader with block cache disabled. HFile.Reader reader = HFile.createReader(fs, storeFilePath, conf); long offset = 0; while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); offset += block.getOnDiskSizeWithHeader(); } }
new HFileBlockIndex.BlockIndexReader(Bytes.BYTES_RAWCOMPARATOR, 1); int sizeToLoadOnOpen = (int) (fileSize - trailer.getLoadOnOpenDataOffset() - trailer.getTrailerSize()); byte[] dataAndMetaIndex = readAllIndex(istream, trailer.getLoadOnOpenDataOffset(), sizeToLoadOnOpen);
/** * Read a storefile in the same manner as a scanner -- using non-positional reads and * without waiting for prefetch to complete. */ private void readStoreFileLikeScanner(Path storeFilePath) throws Exception { // Open the file HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConf, true, conf); do { long offset = 0; while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { HFileBlock block = reader.readBlock(offset, -1, false, /*pread=*/false, false, true, null, null); offset += block.getOnDiskSizeWithHeader(); } } while (!reader.prefetchComplete()); }
private void readStoreFile(Path storeFilePath) throws Exception { // Open the file HFile.Reader reader = HFile.createReader(fs, storeFilePath, cacheConf, true, conf); while (!reader.prefetchComplete()) { // Sleep for a bit Thread.sleep(1000); } // Check that all of the data blocks were preloaded BlockCache blockCache = cacheConf.getBlockCache(); long offset = 0; while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) { HFileBlock block = reader.readBlock(offset, -1, false, true, false, true, null, null); BlockCacheKey blockCacheKey = new BlockCacheKey(reader.getName(), offset); boolean isCached = blockCache.getBlock(blockCacheKey, true, false, true) != null; if (block.getBlockType() == BlockType.DATA || block.getBlockType() == BlockType.ROOT_INDEX || block.getBlockType() == BlockType.INTERMEDIATE_INDEX) { assertTrue(isCached); } offset += block.getOnDiskSizeWithHeader(); } }
private void checkLoadedTrailer(int version, FixedFileTrailer expected, FixedFileTrailer loaded) throws IOException { assertEquals(version, loaded.getMajorVersion()); assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount()); assertEquals(Math.min(expected.getEntryCount(), version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE), loaded.getEntryCount()); if (version == 1) { assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset()); } if (version == 2) { assertEquals(expected.getLastDataBlockOffset(), loaded.getLastDataBlockOffset()); assertEquals(expected.getNumDataIndexLevels(), loaded.getNumDataIndexLevels()); assertEquals(expected.createComparator().getClass().getName(), loaded.createComparator().getClass().getName()); assertEquals(expected.getFirstDataBlockOffset(), loaded.getFirstDataBlockOffset()); assertTrue( expected.createComparator() instanceof CellComparatorImpl); assertEquals(expected.getUncompressedDataIndexSize(), loaded.getUncompressedDataIndexSize()); } assertEquals(expected.getLoadOnOpenDataOffset(), loaded.getLoadOnOpenDataOffset()); assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount()); assertEquals(expected.getTotalUncompressedBytes(), loaded.getTotalUncompressedBytes()); }
while (offset < reader.getTrailer().getLoadOnOpenDataOffset()) {
trailer.getLoadOnOpenDataOffset(), fileSize - trailer.getTrailerSize());