public Scanner(Scan scan) throws IOException { if (scan.getBatch() > 0) { throw new IOException("Batch is not supported in Scanner"); } if (scan.getCaching() <= 0) { scan.setCaching(scannerCaching); } else if (scan.getCaching() == 1 && scan.isReversed()){ // for reverse scan, we need to pass the last row to the next scanner // we need caching number bigger than 1 scan.setCaching(scan.getCaching() + 1); } this.scan = ThriftUtilities.scanFromHBase(scan); }
private Scan setDefaultScanConfig(Scan scan) { // always create a new scan object as we may reset the start row later. Scan newScan = ReflectionUtils.newInstance(scan.getClass(), scan); if (newScan.getCaching() <= 0) { newScan.setCaching(defaultScannerCaching); } if (newScan.getMaxResultSize() <= 0) { newScan.setMaxResultSize(defaultScannerMaxResultSize); } return newScan; }
if (this.scan.getCaching() > 0) { this.caching = this.scan.getCaching(); } else { this.caching = conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING,
/** * Performs a scan of META table for given table. * @param metaTable * @param startRow Where to start the scan * @param stopRow Where to stop the scan * @param type scanned part of meta * @param maxRows maximum rows to return * @param visitor Visitor invoked against each row */ private static CompletableFuture<Void> scanMeta(AsyncTable<AdvancedScanResultConsumer> metaTable, Optional<byte[]> startRow, Optional<byte[]> stopRow, QueryType type, int maxRows, final Visitor visitor) { int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE; Scan scan = getMetaScan(metaTable, rowUpperLimit); for (byte[] family : type.getFamilies()) { scan.addFamily(family); } startRow.ifPresent(scan::withStartRow); stopRow.ifPresent(scan::withStopRow); if (LOG.isDebugEnabled()) { LOG.debug("Scanning META" + " starting at row=" + Bytes.toStringBinary(scan.getStartRow()) + " stopping at row=" + Bytes.toStringBinary(scan.getStopRow()) + " for max=" + rowUpperLimit + " with caching=" + scan.getCaching()); } CompletableFuture<Void> future = new CompletableFuture<Void>(); metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future)); return future; }
private CompletableFuture<OpenScannerResponse> callOpenScanner(HBaseRpcController controller, HRegionLocation loc, ClientService.Interface stub) { boolean isRegionServerRemote = isRemote(loc.getHostname()); incRPCCallsMetrics(scanMetrics, isRegionServerRemote); if (openScannerTries.getAndIncrement() > 1) { incRPCRetriesMetrics(scanMetrics, isRegionServerRemote); } CompletableFuture<OpenScannerResponse> future = new CompletableFuture<>(); try { ScanRequest request = RequestConverter.buildScanRequest(loc.getRegion().getRegionName(), scan, scan.getCaching(), false); stub.scan(controller, request, resp -> { if (controller.failed()) { future.completeExceptionally(controller.getFailed()); return; } future.complete(new OpenScannerResponse(loc, isRegionServerRemote, stub, controller, resp)); }); } catch (IOException e) { future.completeExceptionally(e); } return future; }
private void call() { // As we have a call sequence for scan, it is useless to have a different rpc timeout which is // less than the scan timeout. If the server does not respond in time(usually this will not // happen as we have heartbeat now), we will get an OutOfOrderScannerNextException when // resending the next request and the only way to fix this is to close the scanner and open a // new one. long callTimeoutNs; if (scanTimeoutNs > 0) { long remainingNs = scanTimeoutNs - (System.nanoTime() - nextCallStartNs); if (remainingNs <= 0) { completeExceptionally(true); return; } callTimeoutNs = remainingNs; } else { callTimeoutNs = 0L; } incRPCCallsMetrics(scanMetrics, regionServerRemote); if (tries > 1) { incRPCRetriesMetrics(scanMetrics, regionServerRemote); } resetController(controller, callTimeoutNs); ScanRequest req = RequestConverter.buildScanRequest(scannerId, scan.getCaching(), false, nextCallSeq, false, false, scan.getLimit()); stub.scan(controller, req, resp -> onComplete(controller, resp)); }
int cacheSize = scan2.getCaching(); if (!scan2.getCacheBlocks() || scan2.getCaching() < 2) { scan2.setCacheBlocks(true); cacheSize = 5;
LOG.trace("Scanning META" + " starting at row=" + Bytes.toStringBinary(startRow) + " stopping at row=" + Bytes.toStringBinary(stopRow) + " for max=" + rowUpperLimit + " with caching=" + scan.getCaching());
out.setStartRow(in.getStartRow()); out.setStopRow(in.getStopRow()); out.setCaching(in.getCaching()); out.setMaxVersions(in.getMaxVersions()); for (Map.Entry<byte[], NavigableSet<byte[]>> family : in.getFamilyMap().entrySet()) {
storeLimit = scan.getMaxResultsPerColumnFamily(); storeOffset = scan.getRowOffsetPerColumnFamily(); caching = scan.getCaching(); maxResultSize = scan.getMaxResultSize(); cacheBlocks = scan.getCacheBlocks();
int caching = scan.getCaching(); if (caching > 0) { model.setCaching(caching);
scanBuilder.setConsistency(toConsistency(scan.getConsistency())); if (scan.getCaching() > 0) { scanBuilder.setCaching(scan.getCaching());
scanBuilder.setConsistency(toConsistency(scan.getConsistency())); if (scan.getCaching() > 0) { scanBuilder.setCaching(scan.getCaching());
if (scan.getCaching() <= 0) { scan.setCaching(scannerCaching);
private Scan setDefaultScanConfig(Scan scan) { // always create a new scan object as we may reset the start row later. Scan newScan = ReflectionUtils.newInstance(scan.getClass(), scan); if (newScan.getCaching() <= 0) { newScan.setCaching(defaultScannerCaching); } if (newScan.getMaxResultSize() <= 0) { newScan.setMaxResultSize(defaultScannerMaxResultSize); } return newScan; }
assertEquals(scan.getBatch(), scanCopy.getBatch()); assertEquals(scan.getCacheBlocks(), scanCopy.getCacheBlocks()); assertEquals(scan.getCaching(), scanCopy.getCaching()); assertEquals(scan.getConsistency(), scanCopy.getConsistency()); assertEquals(scan.getFamilies().length, scanCopy.getFamilies().length);
private CompletableFuture<OpenScannerResponse> callOpenScanner(HBaseRpcController controller, HRegionLocation loc, ClientService.Interface stub) { boolean isRegionServerRemote = isRemote(loc.getHostname()); incRPCCallsMetrics(scanMetrics, isRegionServerRemote); if (openScannerTries > 1) { incRPCRetriesMetrics(scanMetrics, isRegionServerRemote); } openScannerTries++; CompletableFuture<OpenScannerResponse> future = new CompletableFuture<>(); try { ScanRequest request = RequestConverter.buildScanRequest(loc.getRegionInfo().getRegionName(), scan, scan.getCaching(), false); stub.scan(controller, request, resp -> { if (controller.failed()) { future.completeExceptionally(controller.getFailed()); return; } future.complete(new OpenScannerResponse(loc, isRegionServerRemote, stub, controller, resp)); }); } catch (IOException e) { future.completeExceptionally(e); } return future; }
private void call() { // As we have a call sequence for scan, it is useless to have a different rpc timeout which is // less than the scan timeout. If the server does not respond in time(usually this will not // happen as we have heartbeat now), we will get an OutOfOrderScannerNextException when // resending the next request and the only way to fix this is to close the scanner and open a // new one. long callTimeoutNs; if (scanTimeoutNs > 0) { long remainingNs = scanTimeoutNs - (System.nanoTime() - nextCallStartNs); if (remainingNs <= 0) { completeExceptionally(true); return; } callTimeoutNs = remainingNs; } else { callTimeoutNs = 0L; } incRPCCallsMetrics(scanMetrics, regionServerRemote); if (tries > 1) { incRPCRetriesMetrics(scanMetrics, regionServerRemote); } resetController(controller, callTimeoutNs); ScanRequest req = RequestConverter.buildScanRequest(scannerId, scan.getCaching(), false, nextCallSeq, false, false, scan.getLimit()); stub.scan(controller, req, resp -> onComplete(controller, resp)); }
@Test public void testScanCaching_Default() throws SQLException { String query = "select * from atable where a_integer=0"; PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class); PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query); QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); Configuration config = HBaseConfiguration.create(); int defaultScannerCacheSize = config.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING); assertEquals(defaultScannerCacheSize, pstmt.getFetchSize()); assertEquals(defaultScannerCacheSize, scan.getCaching()); }
@Test public void testScanCaching_CustomFetchSizeOnStatement() throws SQLException { String query = "select * from atable where a_integer=0"; PhoenixConnection pconn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES)).unwrap(PhoenixConnection.class); PhoenixPreparedStatement pstmt = newPreparedStatement(pconn, query); final int FETCH_SIZE = 25; pstmt.setFetchSize(FETCH_SIZE); QueryPlan plan = pstmt.optimizeQuery(); Scan scan = plan.getContext().getScan(); assertEquals(FETCH_SIZE, pstmt.getFetchSize()); assertEquals(FETCH_SIZE, scan.getCaching()); } }