/** * * @param s * @param k * @return false if not found or if k is after the end. * @throws IOException */ public static boolean seekAtOrAfter(HFileScanner s, Cell k) throws IOException { int result = s.seekTo(k); if(result < 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // Passed KV is smaller than first KV in file, work from start of file return s.seekTo(); } else if(result > 0) { // Passed KV is larger than current KV in file, if there is a next // it is the "after", if not then this scanner is done. return s.next(); } // Seeked to the exact key return true; }
@Override public boolean nextKeyValue() throws IOException, InterruptedException { boolean hasNext; if (!seeked) { LOG.info("Seeking to start"); hasNext = scanner.seekTo(); seeked = true; } else { hasNext = scanner.next(); } if (!hasNext) { return false; } value = scanner.getCell(); count++; return true; }
@Override public boolean seekTo() throws IOException { if (top) { int r = this.delegate.seekTo(splitCell); if (r == HConstants.INDEX_KEY_MAGIC) { return true; } if (r < 0) { // midkey is < first key in file return this.delegate.seekTo(); } if (r > 0) { return this.delegate.next(); } return true; } boolean b = delegate.seekTo(); if (!b) { return b; } // Check key. return (this.delegate.getReader().getComparator().compare(splitCell, getKey())) > 0; }
@Override public Optional<Cell> getFirstKey() { if (!firstKeySeeked) { HFileScanner scanner = getScanner(true, true, false); try { if (scanner.seekTo()) { this.firstKey = Optional.ofNullable(scanner.getKey()); } firstKeySeeked = true; } catch (IOException e) { LOG.warn("Failed seekTo first KV in the file", e); } finally { if(scanner != null) { scanner.close(); } } } return this.firstKey; }
static boolean reseekAtOrAfter(HFileScanner s, Cell k) throws IOException { //This function is similar to seekAtOrAfter function int result = s.reseekTo(k); if (result <= 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // If up to now scanner is not seeked yet, this means passed KV is smaller // than first KV in file, and it is the first time we seek on this file. // So we also need to work from the start of file. if (!s.isSeeked()) { return s.seekTo(); } return true; } // passed KV is larger than current KV in file, if there is a next // it is after, if not then this scanner is done. return s.next(); }
@Override public int seekTo(Cell key) throws IOException { if (top) { if (PrivateCellUtil.compareKeyIgnoresMvcc(getComparator(), key, splitCell) < 0) { return -1; } } else { if (PrivateCellUtil.compareKeyIgnoresMvcc(getComparator(), key, splitCell) >= 0) { // we would place the scanner in the second half. // it might be an error to return false here ever... boolean res = delegate.seekBefore(splitCell); if (!res) { throw new IOException( "Seeking for a key in bottom of file, but key exists in top of file, " + "failed on seekBefore(midkey)"); } return 1; } } return delegate.seekTo(key); }
@Override void setUp() throws Exception { super.setUp(); this.scanner = this.reader.getScanner(false, false); this.scanner.seekTo(); }
private void checkSeekTo(byte[][] keys, HFileScanner scanner, int i) throws IOException { assertEquals("Failed to seek to key #" + i + " (" + Bytes.toStringBinary(keys[i]) + ")", 0, scanner.seekTo(KeyValueUtil.createKeyValueFromKey(keys[i]))); }
private int count() throws IOException { int count = 0; for (HStoreFile f: this.r.stores. get(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); if (!scanner.seekTo()) { continue; } do { count++; } while(scanner.next()); } return count; }
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, true); byte[] gaussianRandomRowBytes = getGaussianRandomRowBytes(); scanner.seekTo(createCell(gaussianRandomRowBytes)); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } // TODO: Fix. Make scanner do Cells. scanner.getCell(); } }
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, true); byte [] b = getRandomRow(); if (scanner.seekTo(createCell(b)) < 0) { LOG.info("Not able to seekTo " + new String(b)); return; } // TODO: Fix scanner so it does Cells Cell c = scanner.getCell(); PerformanceEvaluationCommons.assertKey(b, c); PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); }
private int count() throws IOException { int count = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); if (!scanner.seekTo()) { continue; } do { count++; } while(scanner.next()); } return count; }
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, false); byte [] b = getRandomRow(); // System.out.println("Random row: " + new String(b)); Cell c = createCell(b); if (scanner.seekTo(c) != 0) { LOG.info("Nonexistent row: " + new String(b)); return; } // TODO: HFileScanner doesn't do Cells yet. Temporary fix. c = scanner.getCell(); // System.out.println("Found row: " + // new String(c.getRowArray(), c.getRowOffset(), c.getRowLength())); PerformanceEvaluationCommons.assertKey(b, c); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } c = scanner.getCell(); PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } }
private void metablocks(final String compress) throws Exception { Path mFile = new Path(ROOT_DIR, "meta.hfile"); FSDataOutputStream fout = createFSOutput(mFile); HFileContext meta = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(compress)) .withBlockSize(minBlockSize).build(); Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta) .create(); someTestingWithMetaBlock(writer); writer.close(); fout.close(); FSDataInputStream fin = fs.open(mFile); Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), this.fs.getFileStatus(mFile).getLen(), cacheConf, conf); reader.loadFileInfo(); // No data -- this should return false. assertFalse(reader.getScanner(false, false).seekTo()); someReadingWithMetaBlock(reader); fs.delete(mFile, true); reader.close(); fin.close(); }
private int verifyHFile(Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); assertTrue(count > 0); reader.close(); return count; }
protected void testSeekToInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); FileSystem fs = TEST_UTIL.getTestFileSystem(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount()); HFileScanner scanner = reader.getScanner(false, true); // lies before the start of the file. assertEquals(-1, scanner.seekTo(toKV("a", tagUsage))); assertEquals(1, scanner.seekTo(toKV("d", tagUsage))); assertEquals("c", toRowStr(scanner.getCell())); // Across a block boundary now. // 'h' does not exist so we will get a '1' back for not found. assertEquals(0, scanner.seekTo(toKV("i", tagUsage))); assertEquals("i", toRowStr(scanner.getCell())); assertEquals(1, scanner.seekTo(toKV("l", tagUsage))); assertEquals("k", toRowStr(scanner.getCell())); reader.close(); deleteTestDir(fs); }
/** * Method returns the total KVs in given hfile * @param fs File System * @param p HFile path * @return KV count in the given hfile * @throws IOException */ private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); reader.close(); return count; }
private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf) throws IOException { final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p, cacheConf, bottom, true, new AtomicInteger(0), true, TEST_UTIL.getConfiguration()); halfreader.loadFileInfo(); final HFileScanner scanner = halfreader.getScanner(false, false); scanner.seekTo(); Cell curr; do { curr = scanner.getCell(); KeyValue reseekKv = getLastOnCol(curr); int ret = scanner.reseekTo(reseekKv); assertTrue("reseek to returned: " + ret, ret > 0); // System.out.println(curr + ": " + ret); } while (scanner.next()); int ret = scanner.reseekTo(getLastOnCol(curr)); // System.out.println("Last reseek: " + ret); assertTrue(ret > 0); halfreader.close(true); }
/** * Method returns the total KVs in given hfile * @param fs File System * @param p HFile path * @return KV count in the given hfile * @throws IOException */ private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); reader.close(); return count; } }
private void verifyCounts(int countRow1, int countRow2) throws Exception { int count1 = 0; int count2 = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); scanner.seekTo(); do { byte [] row = CellUtil.cloneRow(scanner.getCell()); if (Bytes.equals(row, STARTROW)) { count1++; } else if(Bytes.equals(row, secondRowBytes)) { count2++; } } while(scanner.next()); } assertEquals(countRow1,count1); assertEquals(countRow2,count2); }