Refine search
public HBaseScanBasedIterator(Table table) { try { Scan scan = new Scan(); scan.setCaching(1000); ResultScanner scanner = table.getScanner(HBaseLookupRowEncoder.CF); scannerIterator = scanner.iterator(); } catch (IOException e) { logger.error("error when scan HBase", e); } }
private static Scan getMetaScan(AsyncTable<?> metaTable, int rowUpperLimit) { Scan scan = new Scan(); int scannerCaching = metaTable.getConfiguration().getInt(HConstants.HBASE_META_SCANNER_CACHING, HConstants.DEFAULT_HBASE_META_SCANNER_CACHING); if (metaTable.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS, HConstants.DEFAULT_USE_META_REPLICAS)) { scan.setConsistency(Consistency.TIMELINE); } if (rowUpperLimit <= scannerCaching) { scan.setLimit(rowUpperLimit); } int rows = Math.min(rowUpperLimit, scannerCaching); scan.setCaching(rows); return scan; }
@Override public Void call() throws Exception { // Configure the scan so that it can read the entire table in a single RPC. We want to test // the case where a scan stops on the server side due to a time limit Scan scan = new Scan(); scan.setMaxResultSize(Long.MAX_VALUE); scan.setCaching(Integer.MAX_VALUE); testEquivalenceOfScanWithHeartbeats(scan, DEFAULT_ROW_SLEEP_TIME, -1, false); return null; } });
private static Scan getMetaScan(Connection connection, int rowUpperLimit) { Scan scan = new Scan(); int scannerCaching = connection.getConfiguration() .getInt(HConstants.HBASE_META_SCANNER_CACHING, HConstants.DEFAULT_HBASE_META_SCANNER_CACHING); if (connection.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS, HConstants.DEFAULT_USE_META_REPLICAS)) { scan.setConsistency(Consistency.TIMELINE); } if (rowUpperLimit > 0) { scan.setLimit(rowUpperLimit); scan.setReadType(Scan.ReadType.PREAD); } scan.setCaching(scannerCaching); return scan; } /**
public void doScan(int n, boolean caching) throws IOException { Scan scan = new Scan(); if (caching) { scan.setCaching(n); } else { scan.setCaching(1); } ResultScanner scanner = table.getScanner(scan); for (int i = 0; i < n; i++) { Result res = scanner.next(); LOG.debug("Result row: " + Bytes.toString(res.getRow()) + ", value: " + Bytes.toString(res.getValue(cf, qualifier))); } }
@Override public Void call() throws Exception { // Configure the scan so that it can read the entire table in a single RPC. We want to test // the case where a scan stops on the server side due to a time limit Scan baseScan = new Scan(); baseScan.setMaxResultSize(Long.MAX_VALUE); baseScan.setCaching(Integer.MAX_VALUE); // Copy the scan before each test. When a scan object is used by a scanner, some of its // fields may be changed such as start row Scan scanCopy = new Scan(baseScan); testEquivalenceOfScanWithHeartbeats(scanCopy, -1, DEFAULT_CF_SLEEP_TIME, false); scanCopy = new Scan(baseScan); testEquivalenceOfScanWithHeartbeats(scanCopy, -1, DEFAULT_CF_SLEEP_TIME, true); return null; } });
public HBaseRecordReader(Connection connection, HBaseSubScan.HBaseSubScanSpec subScanSpec, List<SchemaPath> projectedColumns) { this.connection = connection; hbaseTableName = TableName.valueOf( Preconditions.checkNotNull(subScanSpec, "HBase reader needs a sub-scan spec").getTableName()); hbaseScan = new Scan(subScanSpec.getStartRow(), subScanSpec.getStopRow()); hbaseScanColumnsOnly = new Scan(); hbaseScan .setFilter(subScanSpec.getScanFilter()) .setCaching(TARGET_RECORD_COUNT); setColumns(projectedColumns); }
protected Scan createScanWithSizeLimit() { return new Scan().setMaxResultSize(1).setCaching(Integer.MAX_VALUE).setNeedCursorResult(true); } }
private Scan getScan() { Scan scan = new Scan(); // default scan settings scan.setCacheBlocks(false); scan.setMaxVersions(1); scan.setScanMetricsEnabled(true); if (caching != null) { scan.setCaching(Integer.parseInt(caching)); } return scan; }
Reader() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); table = conn.getTable(htableName); Scan scan = new Scan(); scan.addFamily(CF_B); scan.setCaching(1024); scan.setCacheBlocks(true); scanner = table.getScanner(scan); }
private void initiateScan(HRegion region) throws IOException { Scan scan = new Scan(); scan.setCaching(1); RegionScanner resScanner = null; try { resScanner = region.getScanner(scan); List<Cell> results = new ArrayList<>(); boolean next = resScanner.next(results); try { counter.incrementAndGet(); latch.await(); } catch (InterruptedException e) { } while (next) { next = resScanner.next(results); } } finally { scanCompletedCounter.incrementAndGet(); resScanner.close(); } } }
private void scan(Table table) throws IOException { Scan scan = new Scan(); scan.setCaching(1); scan.setCacheBlocks(false); ResultScanner scanner = table.getScanner(scan); try { scanner.next(); } finally { scanner.close(); } } }
public static Scan buildScan(RawScan rawScan) { Scan scan = new Scan(); scan.setCaching(rawScan.hbaseCaching); scan.setMaxResultSize(rawScan.hbaseMaxResultSize); scan.setCacheBlocks(true); scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); if (rawScan.startKey != null) { scan.setStartRow(rawScan.startKey); } if (rawScan.endKey != null) { scan.setStopRow(rawScan.endKey); } if (rawScan.fuzzyKeys != null) { applyFuzzyFilter(scan, rawScan.fuzzyKeys); } if (rawScan.hbaseColumns != null) { applyHBaseColums(scan, rawScan.hbaseColumns); } return scan; }
/** * Tries to scan a row from passed region */ private void isSuccessfulScan(RegionInfo region) throws IOException { Scan scan = new Scan().withStartRow(region.getStartKey()).setRaw(true).setOneRowLimit() .setMaxResultSize(1L).setCaching(1).setFilter(new FirstKeyOnlyFilter()) .setCacheBlocks(false); try (Table table = conn.getTable(region.getTable()); ResultScanner scanner = table.getScanner(scan)) { scanner.next(); } catch (IOException e) { LOG.error("Could not scan region:" + region.getEncodedName(), e); throw e; } }
private Result getReverseScanResult(Table table, byte[] row, byte[] fam) throws IOException { Scan scan = new Scan(row); scan.setSmall(true); scan.setReversed(true); scan.setCaching(1); scan.addFamily(fam); try (ResultScanner scanner = table.getScanner(scan)) { return scanner.next(); } }
private int doVerify(Path outputDir, int numReducers) throws IOException, InterruptedException, ClassNotFoundException { job = new Job(getConf()); job.setJobName("Link Verifier"); job.setNumReduceTasks(numReducers); job.setJarByClass(getClass()); setJobScannerConf(job); Scan scan = new Scan(); scan.addColumn(FAMILY_NAME, COLUMN_PREV); scan.setCaching(10000); scan.setCacheBlocks(false); String[] split = labels.split(COMMA); scan.setAuthorizations(new Authorizations(split[this.labelIndex * 2], split[(this.labelIndex * 2) + 1])); TableMapReduceUtil.initTableMapperJob(tableName.getName(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); job.getConfiguration().setBoolean("mapreduce.map.speculative", false); job.setReducerClass(VerifyReducer.class); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, outputDir); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
protected Scan createScanWithSparseFilter() { return new Scan().setMaxResultSize(Long.MAX_VALUE).setCaching(Integer.MAX_VALUE) .setNeedCursorResult(true).setAllowPartialResults(true).setFilter(new SparseFilter(false)); }
@Test public void testCrossRowAtomicInRegion() throws IOException, InterruptedException { put(Bytes.toBytes("row1"), CQ1, Bytes.toBytes(1)); put(Bytes.toBytes("row2"), CQ1, Bytes.toBytes(2)); try (Table table = CONN.getTable(tableName); ResultScanner scanner = table.getScanner(new Scan().setCaching(1))) { Result result = scanner.next(); assertArrayEquals(Bytes.toBytes("row1"), result.getRow()); assertEquals(1, Bytes.toInt(result.getValue(CF, CQ1))); move(); put(Bytes.toBytes("row3"), CQ1, Bytes.toBytes(3)); result = scanner.next(); assertArrayEquals(Bytes.toBytes("row2"), result.getRow()); assertEquals(2, Bytes.toInt(result.getValue(CF, CQ1))); assertNull(scanner.next()); } } }
/** * Run through a variety of test configurations with a small scan * @param table * @param reversed * @param rows * @param columns * @throws Exception */ private void testSmallScan(Table table, boolean reversed, int rows, int columns) throws Exception { Scan baseScan = new Scan(); baseScan.setReversed(reversed); baseScan.setSmall(true); Scan scan = new Scan(baseScan); verifyExpectedCounts(table, scan, rows, columns); scan = new Scan(baseScan); scan.setMaxResultSize(1); verifyExpectedCounts(table, scan, rows, columns); scan = new Scan(baseScan); scan.setMaxResultSize(1); scan.setCaching(Integer.MAX_VALUE); verifyExpectedCounts(table, scan, rows, columns); }
public static ReplicationBarrierResult getReplicationBarrierResult(Connection conn, TableName tableName, byte[] row, byte[] encodedRegionName) throws IOException { byte[] metaStartKey = RegionInfo.createRegionName(tableName, row, HConstants.NINES, false); byte[] metaStopKey = RegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW, "", false); Scan scan = new Scan().withStartRow(metaStartKey).withStopRow(metaStopKey) .addColumn(getCatalogFamily(), getRegionStateColumn()) .addFamily(HConstants.REPLICATION_BARRIER_FAMILY).readAllVersions().setReversed(true) .setCaching(10); try (Table table = getMetaHTable(conn); ResultScanner scanner = table.getScanner(scan)) { for (Result result;;) { result = scanner.next(); if (result == null) { return new ReplicationBarrierResult(new long[0], null, Collections.emptyList()); } byte[] regionName = result.getRow(); // TODO: we may look up a region which has already been split or merged so we need to check // whether the encoded name matches. Need to find a way to quit earlier when there is no // record for the given region, for now it will scan to the end of the table. if (!Bytes.equals(encodedRegionName, Bytes.toBytes(RegionInfo.encodeRegionName(regionName)))) { continue; } return getReplicationBarrierResult(result); } } }