private void tuneScanParameters(Scan scan) { scan.setCaching(kylinConfig.getHBaseScanCacheRows()); scan.setMaxResultSize(kylinConfig.getHBaseScanMaxResultSize()); scan.setCacheBlocks(true); }
private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException { // create scan with any properties set from TableInputFormat Scan s = TableInputFormat.createScanFromConfiguration(conf); // Set Scan Versions if (conf.get(TableInputFormat.SCAN_MAXVERSIONS) == null) { // default to all versions unless explicitly set s.setMaxVersions(Integer.MAX_VALUE); } s.setCacheBlocks(false); // Set RowFilter or Prefix Filter if applicable. Filter rowFilter = getRowFilter(args); if (rowFilter!= null) { LOG.info("Setting Row Filter for counter."); s.setFilter(rowFilter); } // Set TimeRange if defined long timeRange[] = getTimeRange(args); if (timeRange != null) { LOG.info("Setting TimeRange for counter."); s.setTimeRange(timeRange[0], timeRange[1]); } return s; }
public void initialize(InputSplit split, Configuration conf) throws IOException { this.scan = TableMapReduceUtil.convertStringToScan(split.getScan()); this.split = split; TableDescriptor htd = split.htd; HRegionInfo hri = this.split.getRegionInfo(); FileSystem fs = FSUtils.getCurrentFileSystem(conf); // region is immutable, this should be fine, // otherwise we have to set the thread read point scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED); // disable caching of data blocks scan.setCacheBlocks(false); scan.setScanMetricsEnabled(true); scanner = new ClientSideRegionScanner(conf, fs, new Path(split.restoreDir), htd, hri, scan, null); }
private Scan getScan() { Scan scan = new Scan(); // default scan settings scan.setCacheBlocks(false); scan.setMaxVersions(1); scan.setScanMetricsEnabled(true); if (caching != null) { scan.setCaching(Integer.parseInt(caching)); } return scan; }
Reader() throws IOException { Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()); table = conn.getTable(htableName); Scan scan = new Scan(); scan.addFamily(CF_B); scan.setCaching(1024); scan.setCacheBlocks(true); scanner = table.getScanner(scan); }
Scan initScan() throws IOException { Scan scan = new Scan(); scan.setCacheBlocks(false); if (startTime != 0 || endTime != 0) { scan.setTimeRange(startTime, endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime); } if (scanBatch > 0) { scan.setBatch(scanBatch); } if (versions >= 0) { scan.setMaxVersions(versions); } if (!isTableStartRow(startRow)) { scan.setStartRow(startRow); } if (!isTableEndRow(stopRow)) { scan.setStopRow(stopRow); } if(families != null) { for(String fam : families.split(",")) { scan.addFamily(Bytes.toBytes(fam)); } } return scan; }
private void scan(Table table) throws IOException { Scan scan = new Scan(); scan.setCaching(1); scan.setCacheBlocks(false); ResultScanner scanner = table.getScanner(scan); try { scanner.next(); } finally { scanner.close(); } } }
scan.setCacheBlocks(false); scan.setCaching(2000); scan.setTimeRange(0, Long.MAX_VALUE);
scan.setCacheBlocks(false); scan.setCaching(2000); scan.setTimeRange(0, Long.MAX_VALUE);
scan.setCacheBlocks(false); scan.setCaching(2000); scan.setTimeRange(0, Long.MAX_VALUE);
public static Scan buildScan(RawScan rawScan) { Scan scan = new Scan(); scan.setCaching(rawScan.hbaseCaching); scan.setMaxResultSize(rawScan.hbaseMaxResultSize); scan.setCacheBlocks(true); scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE)); if (rawScan.startKey != null) { scan.setStartRow(rawScan.startKey); } if (rawScan.endKey != null) { scan.setStopRow(rawScan.endKey); } if (rawScan.fuzzyKeys != null) { applyFuzzyFilter(scan, rawScan.fuzzyKeys); } if (rawScan.hbaseColumns != null) { applyHBaseColums(scan, rawScan.hbaseColumns); } return scan; }
/** * Tries to scan a row from passed region */ private void isSuccessfulScan(RegionInfo region) throws IOException { Scan scan = new Scan().withStartRow(region.getStartKey()).setRaw(true).setOneRowLimit() .setMaxResultSize(1L).setCaching(1).setFilter(new FirstKeyOnlyFilter()) .setCacheBlocks(false); try (Table table = conn.getTable(region.getTable()); ResultScanner scanner = table.getScanner(scan)) { scanner.next(); } catch (IOException e) { LOG.error("Could not scan region:" + region.getEncodedName(), e); throw e; } }
private int doVerify(Path outputDir, int numReducers) throws IOException, InterruptedException, ClassNotFoundException { job = new Job(getConf()); job.setJobName("Link Verifier"); job.setNumReduceTasks(numReducers); job.setJarByClass(getClass()); setJobScannerConf(job); Scan scan = new Scan(); scan.addColumn(FAMILY_NAME, COLUMN_PREV); scan.setCaching(10000); scan.setCacheBlocks(false); String[] split = labels.split(COMMA); scan.setAuthorizations(new Authorizations(split[this.labelIndex * 2], split[(this.labelIndex * 2) + 1])); TableMapReduceUtil.initTableMapperJob(tableName.getName(), scan, VerifyMapper.class, BytesWritable.class, BytesWritable.class, job); TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class); job.getConfiguration().setBoolean("mapreduce.map.speculative", false); job.setReducerClass(VerifyReducer.class); job.setOutputFormatClass(TextOutputFormat.class); TextOutputFormat.setOutputPath(job, outputDir); boolean success = job.waitForCompletion(true); return success ? 0 : 1; }
private void checkTableRows(final Table table, int numRows) throws Exception { Scan scan = new Scan(); scan.setCaching(1); scan.setCacheBlocks(false); ResultScanner scanner = table.getScanner(scan); try { int count = 0; for (int i = 0; i < numRows; ++i) { byte[] row = Bytes.toBytes(String.format("%09d", i)); Result result = scanner.next(); assertTrue(result != null); assertTrue(Bytes.equals(row, result.getRow())); count++; } while (true) { Result result = scanner.next(); if (result == null) break; count++; } assertEquals(numRows, count); } finally { scanner.close(); } } }
region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter());
@Test public void testReverseScanWithPadding() throws Exception { byte[] terminator = new byte[] { -1 }; byte[] row1 = Bytes.add(invert(Bytes.toBytes("a")), terminator); byte[] row2 = Bytes.add(invert(Bytes.toBytes("ab")), terminator); byte[] row3 = Bytes.add(invert(Bytes.toBytes("b")), terminator); Put put1 = new Put(row1); put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put2 = new Put(row2); put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put3 = new Put(row3); put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); region.put(put1); region.put(put2); region.put(put3); region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan); List<Cell> res = new ArrayList<>(); int count = 1; while (scanner.next(res)) { count++; } assertEquals(3, count); }
@Override boolean testRow(final int i) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) .setReadType(opts.scanReadType).setScanMetricsEnabled(true); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(familyName, qualifier); } } else { scan.addFamily(familyName); } } if (opts.filterAll) { scan.setFilter(new FilterAllFilter()); } this.testScanner = table.getScanner(scan); } Result r = testScanner.next(); updateValueSize(r); return true; } }
@Override boolean testRow(final int i) throws IOException { if (this.testScanner == null) { Scan scan = new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching) .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch) .setReadType(opts.scanReadType).setScanMetricsEnabled(true); for (int family = 0; family < opts.families; family++) { byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(familyName, qualifier); } } else { scan.addFamily(familyName); } } if (opts.filterAll) { scan.setFilter(new FilterAllFilter()); } this.testScanner = asyncTable.getScanner(scan); } Result r = testScanner.next(); updateValueSize(r); return true; } }
protected Scan constructScan(byte[] valuePrefix) throws IOException { FilterList list = new FilterList(); Filter filter = new SingleColumnValueFilter(FAMILY_ZERO, COLUMN_ZERO, CompareOperator.EQUAL, new BinaryComparator(valuePrefix)); list.addFilter(filter); if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) .setScanMetricsEnabled(true); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(FAMILY_ZERO, qualifier); } } else { scan.addFamily(FAMILY_ZERO); } scan.setFilter(list); return scan; } }
@Override public boolean evaluate() throws IOException { boolean tableAvailable = getAdmin().isTableAvailable(tableName); if (tableAvailable) { try (Table table = getConnection().getTable(tableName)) { TableDescriptor htd = table.getDescriptor(); for (HRegionLocation loc : getConnection().getRegionLocator(tableName) .getAllRegionLocations()) { Scan scan = new Scan().withStartRow(loc.getRegionInfo().getStartKey()) .withStopRow(loc.getRegionInfo().getEndKey()).setOneRowLimit() .setMaxResultsPerColumnFamily(1).setCacheBlocks(false); for (byte[] family : htd.getColumnFamilyNames()) { scan.addFamily(family); } try (ResultScanner scanner = table.getScanner(scan)) { scanner.next(); } } } } return tableAvailable; } };