@Override public boolean nextKeyValue() throws IOException, InterruptedException { boolean hasNext; if (!seeked) { LOG.info("Seeking to start"); hasNext = scanner.seekTo(); seeked = true; } else { hasNext = scanner.next(); } if (!hasNext) { return false; } value = scanner.getCell(); count++; return true; }
/** * * @param s * @param k * @return false if not found or if k is after the end. * @throws IOException */ public static boolean seekAtOrAfter(HFileScanner s, Cell k) throws IOException { int result = s.seekTo(k); if(result < 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // Passed KV is smaller than first KV in file, work from start of file return s.seekTo(); } else if(result > 0) { // Passed KV is larger than current KV in file, if there is a next // it is the "after", if not then this scanner is done. return s.next(); } // Seeked to the exact key return true; }
@Override public boolean next() throws IOException { if (atEnd) return false; boolean b = delegate.next(); if (!b) { return b; } // constrain the bottom. if (!top) { if (getComparator().compare(splitCell, getKey()) <= 0) { atEnd = true; return false; } } return true; }
@Override public Cell next() throws IOException { Cell retKey = cur; try { // only seek if we aren't at the end. cur == null implies 'end'. if (cur != null) { hfs.next(); setCurrentCell(hfs.getCell()); if (hasMVCCInfo || this.reader.isBulkLoaded()) { skipKVsNewerThanReadpoint(); } } } catch (FileNotFoundException e) { throw e; } catch(IOException e) { throw new IOException("Could not iterate " + this, e); } return retKey; }
static boolean reseekAtOrAfter(HFileScanner s, Cell k) throws IOException { //This function is similar to seekAtOrAfter function int result = s.reseekTo(k); if (result <= 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key return true; } // If up to now scanner is not seeked yet, this means passed KV is smaller // than first KV in file, and it is the first time we seek on this file. // So we also need to work from the start of file. if (!s.isSeeked()) { return s.seekTo(); } return true; } // passed KV is larger than current KV in file, if there is a next // it is after, if not then this scanner is done. return s.next(); }
protected boolean skipKVsNewerThanReadpoint() throws IOException { // We want to ignore all key-values that are newer than our current // readPoint Cell startKV = cur; while(enforceMVCC && cur != null && (cur.getSequenceId() > readPt)) { boolean hasNext = hfs.next(); setCurrentCell(hfs.getCell()); if (hasNext && this.stopSkippingKVsIfNextRow && getComparator().compareRows(cur, startKV) > 0) { return false; } } if (cur == null) { return false; } return true; }
private int count() throws IOException { int count = 0; for (HStoreFile f: this.r.stores. get(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); if (!scanner.seekTo()) { continue; } do { count++; } while(scanner.next()); } return count; }
private int readAndCheckbytes(HFileScanner scanner, int start, int n) throws IOException { String value = "value"; int i = start; for (; i < (start + n); i++) { ByteBuffer key = ByteBuffer.wrap(((KeyValue)scanner.getKey()).getKey()); ByteBuffer val = scanner.getValue(); String keyStr = String.format(localFormatter, Integer.valueOf(i)); String valStr = value + keyStr; KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), Bytes.toBytes("qual"), Bytes.toBytes(valStr)); byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, Bytes.toBytes(key).length).getKey(); assertTrue("bytes for keys do not match " + keyStr + " " + Bytes.toString(Bytes.toBytes(key)), Arrays.equals(kv.getKey(), keyBytes)); byte [] valBytes = Bytes.toBytes(val); assertTrue("bytes for vals do not match " + valStr + " " + Bytes.toString(valBytes), Arrays.equals(Bytes.toBytes(valStr), valBytes)); if (!scanner.next()) { break; } } assertEquals(i, start + n - 1); return (start + n); }
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, true); byte[] gaussianRandomRowBytes = getGaussianRandomRowBytes(); scanner.seekTo(createCell(gaussianRandomRowBytes)); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } // TODO: Fix. Make scanner do Cells. scanner.getCell(); } }
@Override public boolean seekTo() throws IOException { if (top) { int r = this.delegate.seekTo(splitCell); if (r == HConstants.INDEX_KEY_MAGIC) { return true; } if (r < 0) { // midkey is < first key in file return this.delegate.seekTo(); } if (r > 0) { return this.delegate.next(); } return true; } boolean b = delegate.seekTo(); if (!b) { return b; } // Check key. return (this.delegate.getReader().getComparator().compare(splitCell, getKey())) > 0; }
@Override void doRow(int i) throws Exception { if (this.scanner.next()) { // TODO: Fix. Make Scanner do Cells. Cell c = this.scanner.getCell(); PerformanceEvaluationCommons.assertKey(format(i + 1), c); PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } }
private int count() throws IOException { int count = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); if (!scanner.seekTo()) { continue; } do { count++; } while(scanner.next()); } return count; }
do { halfWriter.append(scanner.getCell()); } while (scanner.next());
@Override void doRow(int i) throws Exception { HFileScanner scanner = this.reader.getScanner(false, false); byte [] b = getRandomRow(); // System.out.println("Random row: " + new String(b)); Cell c = createCell(b); if (scanner.seekTo(c) != 0) { LOG.info("Nonexistent row: " + new String(b)); return; } // TODO: HFileScanner doesn't do Cells yet. Temporary fix. c = scanner.getCell(); // System.out.println("Found row: " + // new String(c.getRowArray(), c.getRowOffset(), c.getRowLength())); PerformanceEvaluationCommons.assertKey(b, c); for (int ii = 0; ii < 30; ii++) { if (!scanner.next()) { LOG.info("NOTHING FOLLOWS"); return; } c = scanner.getCell(); PerformanceEvaluationCommons.assertValueSize(c.getValueLength(), ROW_LENGTH); } }
private int verifyHFile(Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(p.getFileSystem(conf), p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); assertTrue(count > 0); reader.close(); return count; }
/** * Method returns the total KVs in given hfile * @param fs File System * @param p HFile path * @return KV count in the given hfile * @throws IOException */ private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); reader.close(); return count; }
private void doTestOfScanAndReseek(Path p, FileSystem fs, Reference bottom, CacheConfig cacheConf) throws IOException { final HalfStoreFileReader halfreader = new HalfStoreFileReader(fs, p, cacheConf, bottom, true, new AtomicInteger(0), true, TEST_UTIL.getConfiguration()); halfreader.loadFileInfo(); final HFileScanner scanner = halfreader.getScanner(false, false); scanner.seekTo(); Cell curr; do { curr = scanner.getCell(); KeyValue reseekKv = getLastOnCol(curr); int ret = scanner.reseekTo(reseekKv); assertTrue("reseek to returned: " + ret, ret > 0); // System.out.println(curr + ": " + ret); } while (scanner.next()); int ret = scanner.reseekTo(getLastOnCol(curr)); // System.out.println("Last reseek: " + ret); assertTrue(ret > 0); halfreader.close(true); }
/** * Method returns the total KVs in given hfile * @param fs File System * @param p HFile path * @return KV count in the given hfile * @throws IOException */ private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException { Configuration conf = util.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), true, conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, false); scanner.seekTo(); int count = 0; do { count++; } while (scanner.next()); reader.close(); return count; } }
HFileScanner s = hsf.getReader().getScanner(false, false); s.seekTo(); while (s.next()) { count++;
private void verifyCounts(int countRow1, int countRow2) throws Exception { int count1 = 0; int count2 = 0; for (HStoreFile f: r.getStore(COLUMN_FAMILY_TEXT).getStorefiles()) { HFileScanner scanner = f.getReader().getScanner(false, false); scanner.seekTo(); do { byte [] row = CellUtil.cloneRow(scanner.getCell()); if (Bytes.equals(row, STARTROW)) { count1++; } else if(Bytes.equals(row, secondRowBytes)) { count2++; } } while(scanner.next()); } assertEquals(countRow1,count1); assertEquals(countRow2,count2); }