key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey()); if ((PrivateCellUtil.compare(topScanner.getReader().getComparator(), midKV, key.array(), key.arrayOffset(), key.limit())) > 0) { fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey()); keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit()); assertTrue(PrivateCellUtil.compare(topScanner.getReader().getComparator(), keyOnlyKV, badmidkey, 0, badmidkey.length) >= 0); if (first) {
bufBackedKeyOnlyKv.setKey(pair.getFirst(), pair.getSecond(), klen); int comp = PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), key, bufBackedKeyOnlyKv); offsetFromPos += klen + vlen; if (this.reader.getFileContext().isIncludesTags()) {
private void populateKV(long nextStartTime, boolean scanSuccessful) { if (!scanSuccessful) { //end of file reached. collect stats and return scanStat.endIteration(0, nextStartTime); return; } prefetchedKey = scan.getKey(); prefetchedValue = scan.getValue(); if (to != null) { // TODO Optimization? Perform int comparison instead of byte[]. Identify // offset of key greater than two. int compare = -1; compare = scan.getReader().getComparator().compare (prefetchedKey.array(), prefetchedKey.arrayOffset(), prefetchedKey.remaining(), to, 0, to.length); if (compare > 0 || (compare == 0 && !toInclusive)) { prefetchedKey = null; prefetchedValue = null; return; } } // account for bytes read and time spent int byteCount = prefetchedKey.remaining() + prefetchedValue.remaining(); scanStat.endIteration(byteCount, nextStartTime); }
@Override public boolean seekTo() throws IOException { if (top) { int r = this.delegate.seekTo(new KeyValue.KeyOnlyKeyValue(splitkey, 0, splitkey.length)); if (r == HConstants.INDEX_KEY_MAGIC) { return true; } if (r < 0) { // midkey is < first key in file return this.delegate.seekTo(); } if (r > 0) { return this.delegate.next(); } return true; } boolean b = delegate.seekTo(); if (!b) { return b; } // Check key. ByteBuffer k = this.delegate.getKey(); return this.delegate.getReader().getComparator(). compareFlatKey(k.array(), k.arrayOffset(), k.limit(), splitkey, 0, splitkey.length) < 0; }
public boolean seekTo() throws IOException { if (top) { int r = this.delegate.seekTo(splitkey); if (r < 0) { // midkey is < first key in file return this.delegate.seekTo(); } if (r > 0) { return this.delegate.next(); } return true; } boolean b = delegate.seekTo(); if (!b) { return b; } // Check key. ByteBuffer k = this.delegate.getKey(); return this.delegate.getReader().getComparator(). compare(k.array(), k.arrayOffset(), k.limit(), splitkey, 0, splitkey.length) < 0; }
public HFileReader() throws IOException { try { FileSystem fs = fsProvider.getFS(); reader = HFile.createReader(fs, path, cacheConf, conf); fileInfo = reader.loadFileInfo(); closed = new AtomicBoolean(false); validate(); if (reader.getComparator() instanceof DelegatingSerializedComparator) { loadComparators((DelegatingSerializedComparator) reader.getComparator()); } // read the old HLL if it exists so that a CardinalityMergeException will trigger a Major Compaction byte[] hll = fileInfo.get(Meta.LOCAL_CARDINALITY_ESTIMATE.toBytes()); if (hll != null) { entryCountEstimate = estimator = HyperLogLog.Builder.build(hll); } else if ((hll = fileInfo.get(Meta.LOCAL_CARDINALITY_ESTIMATE_V2.toBytes())) != null) { entryCountEstimate = estimator = HyperLogLog.Builder.build(hll); } else { estimator = new HyperLogLog(HdfsSortedOplogOrganizer.HLL_CONSTANT); } previousFS = fs; } catch (IOException e) { logger.fine("IO Error while creating reader", e); throw e; } }
public HFileSortedIterator(HFileScanner scan, byte[] from, boolean fromInclusive, byte[] to, boolean toInclusive) throws IOException { this.scan = scan; this.from = from; this.fromInclusive = fromInclusive; this.to = to; this.toInclusive = toInclusive; scanStat = (stats == null) ? new SortedOplogStatistics("", "").new ScanOperation( 0, 0, 0, 0, 0, 0, 0) : stats.getScan(); scanStart = scanStat.begin(); if (scan == null) { return; } assert from == null || to == null || scan.getReader().getComparator().compare(from, to) <= 0; initIterator(); }
public HFileSortedIterator(HFileScanner scan, byte[] from, boolean fromInclusive, byte[] to, boolean toInclusive) throws IOException { this.scan = scan; this.from = from; this.fromInclusive = fromInclusive; this.to = to; this.toInclusive = toInclusive; scanStat = (stats == null) ? new SortedOplogStatistics("", "").new ScanOperation( 0, 0, 0, 0, 0, 0, 0) : stats.getScan(); scanStart = scanStat.begin(); if (scan == null) { return; } assert from == null || to == null || scan.getReader().getComparator().compare( from, 0, from.length, to, 0, to.length) <= 0; initIterator(); }
public KVComparator getComparator() { return reader.getComparator(); }
@Override public SortedIterator<ByteBuffer> scan( byte[] from, boolean fromInclusive, byte[] to, boolean toInclusive, boolean ascending, MetadataFilter filter) throws IOException { if (filter == null || filter.accept(getMetadata(filter.getName()))) { SerializedComparator tmp = (SerializedComparator) reader.getComparator(); tmp = ascending ? tmp : ReversingSerializedComparator.reverse(tmp); // HFileScanner scan = reader.getScanner(true, false, ascending, false); HFileScanner scan = reader.getScanner(true, false, false); return new HFileSortedIterator(scan, tmp, from, fromInclusive, to, toInclusive); } return new BufferIterator(Collections.<byte[], byte[]>emptyMap().entrySet().iterator()); }
public RawComparator<byte []> getComparator() { return reader.getComparator(); }
public BloomFilterImpl() throws IOException { DataInput bin = reader.getGeneralBloomFilterMetadata(); // instantiate bloom filter if meta present in hfile if (bin != null) { hfileBloom = BloomFilterFactory.createFromMeta(bin, reader); if (reader.getComparator() instanceof DelegatingSerializedComparator) { loadComparators((DelegatingSerializedComparator) hfileBloom.getComparator()); } } else { hfileBloom = null; } }
private void setDataBlockEncoder(DataBlockEncoder dataBlockEncoder) { this.dataBlockEncoder = dataBlockEncoder; seeker = dataBlockEncoder.createSeeker(reader.getComparator(), includesMemstoreTS); }
@Override public SerializedComparator getComparator() { return (SerializedComparator) reader.getComparator(); }
public BloomFilterImpl() throws IOException { DataInput bin = reader.getGeneralBloomFilterMetadata(); // instantiate bloom filter if meta present in hfile if (bin != null) { hfileBloom = BloomFilterFactory.createFromMeta(bin, reader); if (reader.getComparator() instanceof DelegatingSerializedComparator) { loadComparators((DelegatingSerializedComparator) hfileBloom.getComparator()); } } else { hfileBloom = null; } }
if (PrivateCellUtil.compareKeyIgnoresMvcc(reader.getComparator(), firstKey, key) >= 0) { long previousBlockOffset = seekToBlock.getPrevBlockOffset();
@Override public int reseekTo(Cell key) throws IOException { int compared; if (isSeeked()) { compared = compareKey(reader.getComparator(), key); if (compared < 1) { // If the required key is less than or equal to current key, then // don't do anything. return compared; } else { // The comparison with no_next_index_key has to be checked if (this.nextIndexedKey != null && (this.nextIndexedKey == KeyValueScanner.NO_NEXT_INDEXED_KEY || PrivateCellUtil .compareKeyIgnoresMvcc(reader.getComparator(), key, nextIndexedKey) < 0)) { // The reader shall continue to scan the current data block instead // of querying the // block index as long as it knows the target key is strictly // smaller than // the next indexed key or the current data block is the last data // block. return loadBlockAndSeekToKey(this.curBlock, nextIndexedKey, false, key, false); } } } // Don't rewind on a reseek operation, because reseek implies that we are // always going forward in the file. return seekTo(key, false); }
@Override public boolean seekTo() throws IOException { if (top) { int r = this.delegate.seekTo(splitCell); if (r == HConstants.INDEX_KEY_MAGIC) { return true; } if (r < 0) { // midkey is < first key in file return this.delegate.seekTo(); } if (r > 0) { return this.delegate.next(); } return true; } boolean b = delegate.seekTo(); if (!b) { return b; } // Check key. return (this.delegate.getReader().getComparator().compare(splitCell, getKey())) > 0; }
public CellComparator getComparator() { return reader.getComparator(); }
public EncodedScanner(HFile.Reader reader, boolean cacheBlocks, boolean pread, boolean isCompaction, HFileContext meta) { super(reader, cacheBlocks, pread, isCompaction); DataBlockEncoding encoding = reader.getDataBlockEncoding(); dataBlockEncoder = encoding.getEncoder(); decodingCtx = dataBlockEncoder.newDataBlockDecodingContext(meta); seeker = dataBlockEncoder.createSeeker( reader.getComparator(), decodingCtx); }