private void checkSeekingConsistency(List<DataBlockEncoder.EncodedSeeker> encodedSeekers, boolean seekBefore, Cell keyValue) { Cell expectedKeyValue = null; ByteBuffer expectedKey = null; ByteBuffer expectedValue = null; for (DataBlockEncoder.EncodedSeeker seeker : encodedSeekers) { seeker.seekToKeyInBlock(keyValue, seekBefore); seeker.rewind(); Cell actualKeyValue = seeker.getCell(); ByteBuffer actualKey = null; actualKey = ByteBuffer.wrap(((KeyValue) seeker.getKey()).getKey()); ByteBuffer actualValue = seeker.getValueShallowCopy(); if (expectedKeyValue != null) { assertTrue(CellUtil.equals(expectedKeyValue, actualKeyValue)); } else { expectedKeyValue = actualKeyValue; } if (expectedKey != null) { assertEquals(expectedKey, actualKey); } else { expectedKey = actualKey; } if (expectedValue != null) { assertEquals(expectedValue, actualValue); } else { expectedValue = actualValue; } } }
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); int i = 0; do { KeyValue expectedKeyValue = sampleKv.get(i); Cell cell = seeker.getCell(); if (PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, expectedKeyValue, cell) != 0) { } while (seeker.next());
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); encodedSeekers.add(seeker);
/** * Updates the current block to be the given {@link HFileBlock}. Seeks to * the the first key/value pair. * * @param newBlock the block to make current * @throws CorruptHFileException */ private void updateCurrentBlock(HFileBlock newBlock) throws CorruptHFileException { block = newBlock; // sanity checks if (block.getBlockType() != BlockType.ENCODED_DATA) { throw new IllegalStateException( "EncodedScanner works only on encoded data blocks"); } short dataBlockEncoderId = block.getDataBlockEncodingId(); if (!DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) { String encoderCls = dataBlockEncoder.getClass().getName(); throw new CorruptHFileException("Encoder " + encoderCls + " doesn't support data block encoding " + DataBlockEncoding.getNameFromId(dataBlockEncoderId)); } seeker.setCurrentBuffer(getEncodedBuffer(newBlock)); blockFetches.incrementAndGet(); // Reset the next indexed key this.nextIndexedKey = null; }
DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); int i = 0; do { KeyValue expectedKeyValue = sampleKv.get(i); Cell cell = seeker.getCell(); if (PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, expectedKeyValue, cell) != 0) { } while (seeker.next());
@Override public boolean seekTo() throws IOException { if (reader == null) { return false; } if (reader.getTrailer().getEntryCount() == 0) { // No data blocks. return false; } long firstDataBlockOffset = reader.getTrailer().getFirstDataBlockOffset(); if (block != null && block.getOffset() == firstDataBlockOffset) { seeker.rewind(); return true; } block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread, isCompaction, true, BlockType.DATA, getEffectiveDataBlockEncoding()); if (block.getOffset() < 0) { throw new IOException("Invalid block offset: " + block.getOffset()); } updateCurrentBlock(block); return true; }
private void checkSeekingConsistency(List<DataBlockEncoder.EncodedSeeker> encodedSeekers, boolean seekBefore, Cell keyValue) { Cell expectedKeyValue = null; ByteBuffer expectedKey = null; ByteBuffer expectedValue = null; for (DataBlockEncoder.EncodedSeeker seeker : encodedSeekers) { seeker.seekToKeyInBlock(keyValue, seekBefore); seeker.rewind(); Cell actualKeyValue = seeker.getCell(); ByteBuffer actualKey = null; actualKey = ByteBuffer.wrap(((KeyValue) seeker.getKey()).getKey()); ByteBuffer actualValue = seeker.getValueShallowCopy(); if (expectedKeyValue != null) { assertTrue(CellUtil.equals(expectedKeyValue, actualKeyValue)); } else { expectedKeyValue = actualKeyValue; } if (expectedKey != null) { assertEquals(expectedKey, actualKey); } else { expectedKey = actualKey; } if (expectedValue != null) { assertEquals(expectedValue, actualValue); } else { expectedValue = actualValue; } } }
private void seekToTheKey(KeyValue expected, List<KeyValue> kvs, Cell toSeek) throws IOException { // create all seekers List<DataBlockEncoder.EncodedSeeker> encodedSeekers = new ArrayList<>(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { if (encoding.getEncoder() == null) { continue; } DataBlockEncoder encoder = encoding.getEncoder(); HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false) .withIncludesMvcc(false).withIncludesTags(false) .withCompression(Compression.Algorithm.NONE).build(); HFileBlockEncodingContext encodingContext = encoder.newDataBlockEncodingContext(encoding, HFILEBLOCK_DUMMY_HEADER, meta); ByteBuffer encodedBuffer = TestDataBlockEncoders.encodeKeyValues(encoding, kvs, encodingContext, this.useOffheapData); DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); encodedSeekers.add(seeker); } // test it! // try a few random seeks checkSeekingConsistency(encodedSeekers, toSeek, expected); }
/** * Updates the current block to be the given {@link HFileBlock}. Seeks to * the the first key/value pair. * * @param newBlock the block to make current */ private void updateCurrentBlock(HFileBlock newBlock) { block = newBlock; // sanity checks if (block.getBlockType() != BlockType.ENCODED_DATA) { throw new IllegalStateException( "EncodedScannerV2 works only on encoded data blocks"); } short dataBlockEncoderId = block.getDataBlockEncodingId(); if (dataBlockEncoder == null || !DataBlockEncoding.isCorrectEncoder(dataBlockEncoder, dataBlockEncoderId)) { DataBlockEncoder encoder = DataBlockEncoding.getDataBlockEncoderById(dataBlockEncoderId); setDataBlockEncoder(encoder); } seeker.setCurrentBuffer(getEncodedBuffer(newBlock)); blockFetches++; }
@Override public boolean seekTo() throws IOException { if (reader == null) { return false; } if (reader.getTrailer().getEntryCount() == 0) { // No data blocks. return false; } long firstDataBlockOffset = reader.getTrailer().getFirstDataBlockOffset(); if (block != null && block.getOffset() == firstDataBlockOffset) { seeker.rewind(); return true; } block = reader.readBlock(firstDataBlockOffset, -1, cacheBlocks, pread, isCompaction, BlockType.DATA); if (block.getOffset() < 0) { throw new IOException("Invalid block offset: " + block.getOffset()); } updateCurrentBlock(block); return true; }
@Test public void testRowIndexWithTagsButNoTagsInCell() throws IOException { List<KeyValue> kvList = new ArrayList<>(); byte[] row = new byte[0]; byte[] family = new byte[0]; byte[] qualifier = new byte[0]; byte[] value = new byte[0]; KeyValue expectedKV = new KeyValue(row, family, qualifier, 1L, Type.Put, value); kvList.add(expectedKV); DataBlockEncoding encoding = DataBlockEncoding.ROW_INDEX_V1; DataBlockEncoder encoder = encoding.getEncoder(); ByteBuffer encodedBuffer = encodeKeyValues(encoding, kvList, getEncodingContext(Algorithm.NONE, encoding), false); HFileContext meta = new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS) .withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build(); DataBlockEncoder.EncodedSeeker seeker = encoder.createSeeker(CellComparatorImpl.COMPARATOR, encoder.newDataBlockDecodingContext(meta)); seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer)); Cell cell = seeker.getCell(); Assert.assertEquals(expectedKV.getLength(), ((KeyValue) cell).getLength()); }
@Override protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey, boolean rewind, byte[] key, int offset, int length, boolean seekBefore) throws IOException { if (block == null || block.getOffset() != seekToBlock.getOffset()) { updateCurrentBlock(seekToBlock); } else if (rewind) { seeker.rewind(); } this.nextIndexedKey = nextIndexedKey; return seeker.seekToKeyInBlock(key, offset, length, seekBefore); } }
@Override public ByteBuffer getKey() { assertValidSeek(); return seeker.getKeyDeepCopy(); }
@Override public Cell getKeyValue() { if (block == null) { return null; } return seeker.getKeyValue(); }
@Override public int compareKey(KVComparator comparator, Cell key) { return seeker.compareKey(comparator, key); } }
@Override public int compareKey(KVComparator comparator, byte[] key, int offset, int length) { return seeker.compareKey(comparator, key, offset, length); }
@Override protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey, boolean rewind, Cell key, boolean seekBefore) throws IOException { if (block == null || block.getOffset() != seekToBlock.getOffset()) { updateCurrentBlock(seekToBlock); } else if (rewind) { seeker.rewind(); } this.nextIndexedKey = nextIndexedKey; return seeker.seekToKeyInBlock(key, seekBefore); }
@Override public ByteBuffer getValue() { assertValidSeek(); return seeker.getValueShallowCopy(); }
@Override public KeyValue getKeyValue() { if (block == null) { return null; } return seeker.getKeyValue(); }
@Override public ByteBuffer getKey() { assertValidSeek(); return seeker.getKeyDeepCopy(); }