private List<Cell> performScan(byte[] row1, byte[] fam1) throws IOException { Scan scan = new Scan().withStartRow(row1).addFamily(fam1).readVersions(MAX_VERSIONS); List<Cell> actual = new ArrayList<>(); InternalScanner scanner = region.getScanner(scan); boolean hasNext = scanner.next(actual); assertEquals(false, hasNext); return actual; }
private void verifyScan(Scan s, long expectedRows, long expectedKeys) throws IOException { InternalScanner scanner = this.region.getScanner(s); List<Cell> results = new ArrayList<>(); int i = 0; for (boolean done = true; done; i++) { done = scanner.next(results); Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.getInstance()); LOG.info("counter=" + i + ", " + results); if (results.isEmpty()) break; assertTrue("Scanned too many rows! Only expected " + expectedRows + " total but already scanned " + (i+1), expectedRows > i); assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + results.size(), expectedKeys, results.size()); results.clear(); } assertEquals("Expected " + expectedRows + " rows but scanned " + i + " rows", expectedRows, i); }
public static void doScan(HRegion region, Scan scan, List<Cell> result) throws IOException { InternalScanner scanner = null; try { scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED); scanner = region.getScanner(scan); result.clear(); scanner.next(result); } finally { if (scanner != null) { scanner.close(); } } } }
private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException { String cf = "f"; Scan scan = new Scan(); scan.addFamily(cf.getBytes()); scan.setFilter(filter); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes()); HRegion first = regions.get(0); first.getScanner(scan); RegionScanner scanner = first.getScanner(scan); List<Cell> results = new ArrayList<>(); // Result result; long timeBeforeScan = System.currentTimeMillis(); int found = 0; while (scanner.next(results)) { found += results.size(); results.clear(); } found += results.size(); long scanTime = System.currentTimeMillis() - timeBeforeScan; scanner.close(); LOG.info("\nscan time = " + scanTime + "ms"); LOG.info("found " + found + " results\n"); assertEquals(expectedSize, found); }
/** * This method tests https://issues.apache.org/jira/browse/HBASE-2516. * * @throws IOException */ @Test public void testGetScanner_WithRegionClosed() throws IOException { byte[] fam1 = Bytes.toBytes("fam1"); byte[] fam2 = Bytes.toBytes("fam2"); byte[][] families = { fam1, fam2 }; // Setting up region try { this.region = initHRegion(tableName, method, CONF, families); } catch (IOException e) { e.printStackTrace(); fail("Got IOException during initHRegion, " + e.getMessage()); } region.closed.set(true); try { region.getScanner(null); fail("Expected to get an exception during getScanner on a region that is closed"); } catch (NotServingRegionException e) { // this is the correct exception that is expected } catch (IOException e) { fail("Got wrong type of exception - should be a NotServingRegionException, " + "but was an IOException: " + e.getMessage()); } }
@Test public void testCoprocessorInterface() throws IOException { TableName tableName = TableName.valueOf(name.getMethodName()); byte [][] families = { fam1, fam2, fam3 }; Configuration hc = initConfig(); HRegion region = initHRegion(tableName, name.getMethodName(), hc, new Class<?>[]{CoprocessorImpl.class}, families); for (int i = 0; i < 3; i++) { HBaseTestCase.addContent(region, fam3); region.flush(true); } region.compact(false); // HBASE-4197 Scan s = new Scan(); RegionScanner scanner = region.getCoprocessorHost().postScannerOpen(s, region.getScanner(s)); assertTrue(scanner instanceof CustomScanner); // this would throw an exception before HBASE-4197 scanner.next(new ArrayList<>()); HBaseTestingUtility.closeRegionAndWAL(region); Coprocessor c = region.getCoprocessorHost().findCoprocessor(CoprocessorImpl.class); assertTrue("Coprocessor not started", ((CoprocessorImpl)c).wasStarted()); assertTrue("Coprocessor not stopped", ((CoprocessorImpl)c).wasStopped()); assertTrue(((CoprocessorImpl)c).wasOpened()); assertTrue(((CoprocessorImpl)c).wasClosed()); assertTrue(((CoprocessorImpl)c).wasFlushed()); assertTrue(((CoprocessorImpl)c).wasCompacted()); }
private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) throws IOException { // Defaults FilterList.Operator.MUST_PASS_ALL. FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); // Only return rows where this column value exists in the row. SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return r.getScanner(scan); }
private int countDeleteMarkers(HRegion region) throws IOException { Scan s = new Scan(); s.setRaw(true); // use max versions from the store(s) s.setMaxVersions(region.getStores().iterator().next().getScanInfo().getMaxVersions()); InternalScanner scan = region.getScanner(s); List<Cell> kvs = new ArrayList<>(); int res = 0; boolean hasMore; do { hasMore = scan.next(kvs); for (Cell kv : kvs) { if(CellUtil.isDelete(kv)) res++; } kvs.clear(); } while (hasMore); scan.close(); return res; }
private void initiateScan(HRegion region) throws IOException { Scan scan = new Scan(); scan.setCaching(1); RegionScanner resScanner = null; try { resScanner = region.getScanner(scan); List<Cell> results = new ArrayList<>(); boolean next = resScanner.next(results); try { counter.incrementAndGet(); latch.await(); } catch (InterruptedException e) { } while (next) { next = resScanner.next(results); } } finally { scanCompletedCounter.incrementAndGet(); resScanner.close(); } } }
/** * The following test is to ensure old(such as hbase0.94) filterRow() can be correctly fired in * 0.96+ code base. * * See HBASE-10366 * * @throws Exception */ @Test public void test94FilterRowCompatibility() throws Exception { Scan s = new Scan(); OldTestFilter filter = new OldTestFilter(); s.setFilter(filter); InternalScanner scanner = this.region.getScanner(s); ArrayList<Cell> values = new ArrayList<>(); scanner.next(values); assertTrue("All rows should be filtered out", values.isEmpty()); }
private void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys) throws IOException { InternalScanner scanner = this.region.getScanner(s); List<Cell> results = new ArrayList<>(); int i = 0; for (boolean done = true; done; i++) { done = scanner.next(results); Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.getInstance()); LOG.info("counter=" + i + ", " + results); if(results.isEmpty()) break; assertTrue("Scanned too many rows! Only expected " + expectedRows + " total but already scanned " + (i+1), expectedRows > i); assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + results.size(), expectedKeys, results.size()); results.clear(); } assertEquals("Expected " + expectedRows + " rows but scanned " + i + " rows", expectedRows, i); }
@Override public Object run() throws Exception { runWALSplit(newConf); WAL wal2 = createWAL(newConf, hbaseRootDir, logName); HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, hri, htd, wal2); long seqid2 = region2.getOpenSeqNum(); assertTrue(seqid2 > -1); assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan()))); // I can't close wal1. Its been appropriated when we split. region2.close(); wal2.close(); return null; } });
protected void assertScan(final HRegion r, final byte[] fs, final byte[] firstValue) throws IOException { byte[][] families = { fs }; Scan scan = new Scan(); for (int i = 0; i < families.length; i++) scan.addFamily(families[i]); InternalScanner s = r.getScanner(scan); try { List<Cell> curVals = new ArrayList<>(); boolean first = true; OUTER_LOOP: while (s.next(curVals)) { for (Cell kv : curVals) { byte[] val = CellUtil.cloneValue(kv); byte[] curval = val; if (first) { first = false; assertTrue(Bytes.compareTo(curval, firstValue) == 0); } else { // Not asserting anything. Might as well break. break OUTER_LOOP; } } } } finally { s.close(); } }
public List<Cell> get(Get get, boolean withCoprocessor, long nonceGroup, long nonce) throws IOException { List<Cell> results = new ArrayList<>(); long before = EnvironmentEdgeManager.currentTime(); // pre-get CP hook if (withCoprocessor && (coprocessorHost != null)) { if (coprocessorHost.preGet(get, results)) { metricsUpdateForGet(results, before); return results; } } Scan scan = new Scan(get); if (scan.getLoadColumnFamiliesOnDemandValue() == null) { scan.setLoadColumnFamiliesOnDemand(isLoadingCfsOnDemandDefault()); } RegionScanner scanner = null; try { scanner = getScanner(scan, null, nonceGroup, nonce); scanner.next(results); } finally { if (scanner != null) scanner.close(); } // post-get CP hook if (withCoprocessor && (coprocessorHost != null)) { coprocessorHost.postGet(get, results); } metricsUpdateForGet(results, before); return results; }
void rowInclusiveStopFilter(Scan scan, byte[] stopRow) throws IOException { List<Cell> results = new ArrayList<>(); scan.addFamily(HConstants.CATALOG_FAMILY); InternalScanner s = region.getScanner(scan); boolean hasMore = true; while (hasMore) { hasMore = s.next(results); for (Cell kv : results) { assertTrue(Bytes.compareTo(CellUtil.cloneRow(kv), stopRow) <= 0); } results.clear(); } s.close(); }
@Test public void testGetScanner_WithNotOkFamilies() throws IOException { byte[] fam1 = Bytes.toBytes("fam1"); byte[] fam2 = Bytes.toBytes("fam2"); byte[][] families = { fam1 }; // Setting up region this.region = initHRegion(tableName, method, CONF, families); Scan scan = new Scan(); scan.addFamily(fam2); boolean ok = false; try { region.getScanner(scan); } catch (Exception e) { ok = true; } assertTrue("Families could not be found in Region", ok); }
private void verifyScanNoEarlyOut(Scan s, long expectedRows, long expectedKeys) throws IOException { InternalScanner scanner = this.region.getScanner(s); List<Cell> results = new ArrayList<>(); int i = 0; for (boolean done = true; done; i++) { done = scanner.next(results); Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.getInstance()); LOG.info("counter=" + i + ", " + results); if(results.isEmpty()) { break; } assertTrue("Scanned too many rows! Only expected " + expectedRows + " total but already scanned " + (i+1), expectedRows > i); assertEquals("Expected " + expectedKeys + " keys per row but " + "returned " + results.size(), expectedKeys, results.size()); results.clear(); } assertEquals("Expected " + expectedRows + " rows but scanned " + i + " rows", expectedRows, i); } }
@Override public Object run() throws Exception { runWALSplit(newConf); WAL wal2 = createWAL(newConf, hbaseRootDir, logName); HRegion region2 = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir, hri, htd, wal2); long seqid2 = region2.getOpenSeqNum(); assertTrue(seqid2 > -1); assertEquals(rowsInsertedCount, getScannedCount(region2.getScanner(new Scan()))); // I can't close wal1. Its been appropriated when we split. region2.close(); wal2.close(); return null; } });
private int countMobRows() throws IOException { Scan scan = new Scan(); // Do not retrieve the mob data when scanning scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE)); InternalScanner scanner = region.getScanner(scan); int scannedCount = 0; List<Cell> results = new ArrayList<>(); boolean hasMore = true; while (hasMore) { hasMore = scanner.next(results); for (Cell c : results) { if (MobUtils.isMobReferenceCell(c)) { scannedCount++; } } results.clear(); } scanner.close(); return scannedCount; }
void rowPrefixFilter(Scan scan) throws IOException { List<Cell> results = new ArrayList<>(); scan.addFamily(HConstants.CATALOG_FAMILY); InternalScanner s = region.getScanner(scan); boolean hasMore = true; while (hasMore) { hasMore = s.next(results); for (Cell kv : results) { assertEquals((byte)'a', CellUtil.cloneRow(kv)[0]); assertEquals((byte)'b', CellUtil.cloneRow(kv)[1]); } results.clear(); } s.close(); }