@Override public boolean nextKeyValue() throws IOException, InterruptedException { boolean hasNext; if (!seeked) { LOG.info("Seeking to start"); hasNext = scanner.seekTo(); seeked = true; } else { hasNext = scanner.next(); } if (!hasNext) { return false; } value = scanner.getCell(); count++; return true; }
@Override public Optional<Cell> getFirstKey() { if (!firstKeySeeked) { HFileScanner scanner = getScanner(true, true, false); try { if (scanner.seekTo()) { this.firstKey = Optional.ofNullable(scanner.getKey()); } firstKeySeeked = true; } catch (IOException e) { LOG.warn("Failed seekTo first KV in the file", e); } finally { if(scanner != null) { scanner.close(); } } } return this.firstKey; }
@Override public boolean seekTo() throws IOException { if (top) { int r = this.delegate.seekTo(splitCell); if (r == HConstants.INDEX_KEY_MAGIC) { return true; } if (r < 0) { // midkey is < first key in file return this.delegate.seekTo(); } if (r > 0) { return this.delegate.next(); } return true; } boolean b = delegate.seekTo(); if (!b) { return b; } // Check key. return (this.delegate.getReader().getComparator().compare(splitCell, getKey())) > 0; }
static boolean reseekAtOrAfter(HFileScanner s, Cell k) throws IOException { //This function is similar to seekAtOrAfter function int result = s.reseekTo(k); if (result <= 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // If up to now scanner is not seeked yet, this means passed KV is smaller // than first KV in file, and it is the first time we seek on this file. // So we also need to work from the start of file. if (!s.isSeeked()) { return s.seekTo(); } return true; } // passed KV is larger than current KV in file, if there is a next // it is after, if not then this scanner is done. return s.next(); }
/** * * @param s * @param k * @return false if not found or if k is after the end. * @throws IOException */ public static boolean seekAtOrAfter(HFileScanner s, Cell k) throws IOException { int result = s.seekTo(k); if(result < 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // Passed KV is smaller than first KV in file, work from start of file return s.seekTo(); } else if(result > 0) { // Passed KV is larger than current KV in file, if there is a next // it is the "after", if not then this scanner is done. return s.next(); } // Seeked to the exact key return true; }
ByteBuffer key = null; HFileScanner topScanner = top.getScanner(false, false); while ((!topScanner.isSeeked() && topScanner.seekTo()) || (topScanner.isSeeked() && topScanner.next())) { key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey()); if ((PrivateCellUtil.compare(topScanner.getReader().getComparator(), midKV, key.array(), key.arrayOffset(), key.limit())) > 0) { fail("key=" + Bytes.toStringBinary(key) + " < midkey=" + while ((!bottomScanner.isSeeked() && bottomScanner.seekTo()) || bottomScanner.next()) { previous = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey()); key = ByteBuffer.wrap(((KeyValue) bottomScanner.getKey()).getKey()); if (first) { first = false; topScanner = top.getScanner(false, false); KeyValue.KeyOnlyKeyValue keyOnlyKV = new KeyValue.KeyOnlyKeyValue(); while ((!topScanner.isSeeked() && topScanner.seekTo()) || topScanner.next()) { key = ByteBuffer.wrap(((KeyValue) topScanner.getKey()).getKey()); keyOnlyKV.setKey(key.array(), 0 + key.arrayOffset(), key.limit()); assertTrue(PrivateCellUtil.compare(topScanner.getReader().getComparator(), keyOnlyKV, badmidkey, 0, badmidkey.length) >= 0); if (first) { first = true; bottomScanner = bottom.getScanner(false, false);
scanner.seekTo(); readAllRecords(scanner); int seekTo = scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))); System.out.println(seekTo); assertTrue("location lookup failed", scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(50))) == 0); ByteBuffer readKey = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), Bytes.toBytes(readKey))); scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); ByteBuffer val1 = scanner.getValue(); scanner.seekTo(KeyValueUtil.createKeyValueFromKey(getSomeKey(0))); ByteBuffer val2 = scanner.getValue(); assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2)));
private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf) throws IOException { final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p, cacheConf, bottom, true, new AtomicInteger(0), true, TEST_UTIL.getConfiguration()); halfreader.loadFileInfo(); final HFileScanner scanner = halfreader.getScanner(false, false); scanner.seekTo(); Cell curr; do { curr = scanner.getCell(); KeyValue reseekKv = getLastOnCol(curr); int ret = scanner.reseekTo(reseekKv); assertTrue("reseek to returned: " + ret, ret > 0); // System.out.println(curr + ": " + ret); } while (scanner.next()); int ret = scanner.reseekTo(getLastOnCol(curr)); // System.out.println("Last reseek: " + ret); assertTrue(ret > 0); halfreader.close(true); }
public static void doSmokeTest(FileSystem fs, Path path, String codec) throws Exception { Configuration conf = HBaseConfiguration.create(); HFileContext context = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(codec)).build(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf) .withPath(fs, path) .withFileContext(context) .create(); // Write any-old Cell... final byte [] rowKey = Bytes.toBytes("compressiontestkey"); Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval")); writer.append(c); writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval")); writer.close(); Cell cc = null; HFile.Reader reader = HFile.createReader(fs, path, CacheConfig.DISABLED, true, conf); try { reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); scanner.seekTo(); // position to the start of file // Scanner does not do Cells yet. Do below for now till fixed. cc = scanner.getCell(); if (CellComparator.getInstance().compareRows(c, cc) != 0) { throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString()); } } finally { reader.close(); } }
private int readAndCheckbytes(HFileScanner scanner, int start, int n) throws IOException { String value = "value"; int i = start; for (; i < (start + n); i++) { ByteBuffer key = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); ByteBuffer val = scanner.getValue(); String keyStr = String.format(localFormatter, Integer.valueOf(i)); String valStr = value + keyStr; KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), Bytes.toBytes("qual"), Bytes.toBytes(valStr)); byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, Bytes.toBytes(key).length).getKey(); assertTrue("bytes for keys do not match " + keyStr + " " + Bytes.toString(Bytes.toBytes(key)), Arrays.equals(kv.getKey(), keyBytes)); byte [] valBytes = Bytes.toBytes(val); assertTrue("bytes for vals do not match " + valStr + " " + Bytes.toString(valBytes), Arrays.equals(Bytes.toBytes(valStr), valBytes)); if (!scanner.next()) { break; } } assertEquals(i, start + n - 1); return (start + n); }
@Override public Cell next() throws IOException { Cell retKey = cur; try { // only seek if we aren't at the end. cur == null implies 'end'. if (cur != null) { hfs.next(); setCurrentCell(hfs.getCell()); if (hasMVCCInfo || this.reader.isBulkLoaded()) { skipKVsNewerThanReadpoint(); } } } catch (FileNotFoundException e) { throw e; } catch(IOException e) { throw new IOException("Could not iterate " + this, e); } return retKey; }
HFileScanner scanner = reader.getScanner(false, true); scanner.seekTo(); for (int i = 0; i < keyList.size(); i++) { Integer key = keyList.get(i); String value = valueList.get(i); long start = System.nanoTime(); scanner.seekTo(new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes .toBytes("qual"), Bytes.toBytes(value))); assertEquals(value, scanner.getValueString()); scanner.seekTo(); for (int i = 0; i < keyList.size(); i += 10) { Integer key = keyList.get(i); String value = valueList.get(i); long start = System.nanoTime(); scanner.reseekTo(new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes .toBytes("qual"), Bytes.toBytes(value))); assertEquals("i is " + i, value, scanner.getValueString());
@Override public int seekTo(Cell key) throws IOException { if (top) { if (PrivateCellUtil.compareKeyIgnoresMvcc(getComparator(), key, splitCell) < 0) { return -1; } } else { if (PrivateCellUtil.compareKeyIgnoresMvcc(getComparator(), key, splitCell) >= 0) { // we would place the scanner in the second half. // it might be an error to return false here ever... boolean res = delegate.seekBefore(splitCell); if (!res) { throw new IOException( "Seeking for a key in bottom of file, but key exists in top of file, " + "failed on seekBefore(midkey)"); } return 1; } } return delegate.seekTo(key); }
private void populateKV(long nextStartTime, boolean scanSuccessful) { if (!scanSuccessful) { //end of file reached. collect stats and return scanStat.endIteration(0, nextStartTime); return; } prefetchedKey = scan.getKey(); prefetchedValue = scan.getValue(); if (to != null) { // TODO Optimization? Perform int comparison instead of byte[]. Identify // offset of key greater than two. int compare = -1; compare = scan.getReader().getComparator().compare (prefetchedKey.array(), prefetchedKey.arrayOffset(), prefetchedKey.remaining(), to, 0, to.length); if (compare > 0 || (compare == 0 && !toInclusive)) { prefetchedKey = null; prefetchedValue = null; return; } } // account for bytes read and time spent int byteCount = prefetchedKey.remaining() + prefetchedValue.remaining(); scanStat.endIteration(byteCount, nextStartTime); }
@Override public byte[] getFirstKey() { if (!firstKeySeeked) { HFileScanner scanner = getScanner(true, true, false); try { if (scanner.seekTo()) { this.firstKey = Bytes.toBytes(scanner.getKey()); } firstKeySeeked = true; } catch (IOException e) { LOG.warn("Failed seekTo first KV in the file", e); } } return this.firstKey; } }
Cell seekKey = PrivateCellUtil.createFirstOnRow(key); if (seekCount != null) seekCount.increment(); if (!hfs.seekBefore(seekKey)) { this.cur = null; return false; Cell curCell = hfs.getCell(); Cell firstKeyOfPreviousRow = PrivateCellUtil.createFirstOnRow(curCell); setCurrentCell(hfs.getCell()); this.stopSkippingKVsIfNextRow = true; boolean resultOfSkipKVs;
reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); assertFalse(scanner.seekBefore(toKV("a", tagUsage))); assertFalse(scanner.seekBefore(toKV("b", tagUsage))); assertFalse(scanner.seekBefore(toKV("c", tagUsage))); assertTrue(scanner.seekBefore(toKV("d", tagUsage))); assertFalse(scanner.getCell() instanceof ByteBufferKeyValue); assertEquals("c", toRowStr(scanner.getCell())); assertEquals(0, scanner.reseekTo(toKV("c", tagUsage))); assertEquals("c", toRowStr(scanner.getCell())); assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getCell())); assertTrue(scanner.seekBefore(toKV("e", tagUsage))); assertEquals("c", toRowStr(scanner.getCell())); assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); assertEquals("e", toRowStr(scanner.getCell())); assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getCell())); assertTrue(scanner.seekBefore(toKV("f", tagUsage))); assertEquals("e", toRowStr(scanner.getCell())); assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); assertEquals("e", toRowStr(scanner.getCell())); assertEquals(0, scanner.reseekTo(toKV("g", tagUsage)));
@Override void setUp() throws Exception { super.setUp(); this.scanner = this.reader.getScanner(false, false); this.scanner.seekTo(); }