/** * @param scan the HBase scan object to use to read data from HBase * @param canFamilyBeAbsent whether column family can be absent in familyMap of scan */ private static void validateParameters(Scan scan, boolean canFamilyBeAbsent) throws IOException { if (scan == null || (Bytes.equals(scan.getStartRow(), scan.getStopRow()) && !Bytes.equals(scan.getStartRow(), HConstants.EMPTY_START_ROW)) || ((Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) > 0) && !Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW))) { throw new IOException("Agg client Exception: Startrow should be smaller than Stoprow"); } else if (!canFamilyBeAbsent) { if (scan.getFamilyMap().size() != 1) { throw new IOException("There must be only one family."); } } }
public ResultScanner getScanner(Scan scan) throws IOException { final List<Result> ret = new ArrayList<Result>(); byte[] st = scan.getStartRow(); byte[] sp = scan.getStopRow(); Filter filter = scan.getFilter();
@Test public void testScan() throws Exception { Scan desScan = ProtobufUtil.toScan(scanProto); assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow())); assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow())); assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks()); Set<byte[]> set = null; scanProto = ProtobufUtil.toScan(scan); desScan = ProtobufUtil.toScan(scanProto); Filter f = desScan.getFilter(); assertTrue(f instanceof PrefixFilter);
@Test public void testSubstrSetScanKey() throws Exception { String query = "SELECT inst FROM ptsdb WHERE substr(inst, 0, 3) = 'abc'"; List<Object> binds = Collections.emptyList(); Scan scan = compileQuery(query, binds); assertArrayEquals(Bytes.toBytes("abc"), scan.getStartRow()); assertArrayEquals(ByteUtil.nextKey(Bytes.toBytes("abc")), scan.getStopRow()); assertTrue(scan.getFilter() == null); // Extracted. }
/** * A method for checking Bloom filters. Called directly from * StoreFileScanner in case of a multi-column query. * * @return True if passes */ private boolean passesGeneralRowPrefixBloomFilter(Scan scan) { BloomFilter bloomFilter = this.generalBloomFilter; if (bloomFilter == null) { return true; } byte[] row = scan.getStartRow(); byte[] rowPrefix; if (scan.isGetScan()) { rowPrefix = Bytes.copy(row, 0, Math.min(prefixLength, row.length)); } else { // For non-get scans // Find out the common prefix of startRow and stopRow. int commonLength = Bytes.findCommonPrefix(scan.getStartRow(), scan.getStopRow(), scan.getStartRow().length, scan.getStopRow().length, 0, 0); // startRow and stopRow don't have the common prefix. // Or the common prefix length is less than prefixLength if (commonLength <= 0 || commonLength < prefixLength) { return true; } rowPrefix = Bytes.copy(row, 0, prefixLength); } return checkGeneralBloomFilter(rowPrefix, null, bloomFilter); }
public static TScan scanFromHBase(Scan in) throws IOException { TScan out = new TScan(); out.setStartRow(in.getStartRow()); out.setStopRow(in.getStopRow()); out.setCaching(in.getCaching()); out.setMaxVersions(in.getMaxVersions()); if (in.getFilter() != null) { try { out.setFilterBytes(in.getFilter().toByteArray()); } catch (IOException ioE) { throw new RuntimeException(ioE);
@Test public void testRTrimSetScanKey() throws Exception { String query = "SELECT inst FROM ptsdb WHERE rtrim(inst) = 'abc'"; List<Object> binds = Collections.emptyList(); Scan scan = compileQuery(query, binds); assertArrayEquals(Bytes.toBytes("abc"), scan.getStartRow()); assertArrayEquals(ByteUtil.nextKey(Bytes.toBytes("abc ")), scan.getStopRow()); assertNotNull(scan.getFilter()); }
/** * Checks whether the given scan rowkey range overlaps with the current storefile's * @param scan the scan specification. Used to determine the rowkey range. * @return true if there is overlap, false otherwise */ public boolean passesKeyRangeFilter(Scan scan) { Optional<Cell> firstKeyKV = this.getFirstKey(); Optional<Cell> lastKeyKV = this.getLastKey(); if (!firstKeyKV.isPresent() || !lastKeyKV.isPresent()) { // the file is empty return false; } if (Bytes.equals(scan.getStartRow(), HConstants.EMPTY_START_ROW) && Bytes.equals(scan.getStopRow(), HConstants.EMPTY_END_ROW)) { return true; } byte[] smallestScanRow = scan.isReversed() ? scan.getStopRow() : scan.getStartRow(); byte[] largestScanRow = scan.isReversed() ? scan.getStartRow() : scan.getStopRow(); boolean nonOverLapping = (getComparator() .compareRows(firstKeyKV.get(), largestScanRow, 0, largestScanRow.length) > 0 && !Bytes.equals(scan.isReversed() ? scan.getStartRow() : scan.getStopRow(), HConstants.EMPTY_END_ROW)) || getComparator().compareRows(lastKeyKV.get(), smallestScanRow, 0, smallestScanRow.length) < 0; return !nonOverLapping; }
startRow = scan.getStartRow(); includeStartRow = scan.includeStartRow(); stopRow = scan.getStopRow(); includeStopRow = scan.includeStopRow(); maxVersions = scan.getMaxVersions(); maxResultSize = scan.getMaxResultSize(); cacheBlocks = scan.getCacheBlocks(); filter = scan.getFilter(); // clone? loadColumnFamiliesOnDemand = scan.getLoadColumnFamiliesOnDemandValue(); consistency = scan.getConsistency();
@Test public void testMultipleNonEqualitiesPkColumn() throws SQLException { String tenantId = "000000000000001"; String keyPrefix = "002"; String query = "select * from atable where organization_id >= '" + tenantId + "' AND substr(entity_id,1,3) > '" + keyPrefix + "'"; Scan scan = new Scan(); List<Object> binds = Collections.emptyList(); compileStatement(query, scan, binds); assertNotNull(scan.getFilter()); assertArrayEquals(PDataType.VARCHAR.toBytes(tenantId), scan.getStartRow()); assertArrayEquals(HConstants.EMPTY_END_ROW, scan.getStopRow()); }
@Override public void updateReaders(List<HStoreFile> sfs, List<KeyValueScanner> memStoreScanners) throws IOException { if (CollectionUtils.isEmpty(sfs) && CollectionUtils.isEmpty(memStoreScanners)) { return; } flushLock.lock(); try { flushed = true; final boolean isCompaction = false; boolean usePread = get || scanUsePread; // SEE HBASE-19468 where the flushed files are getting compacted even before a scanner // calls next(). So its better we create scanners here rather than next() call. Ensure // these scanners are properly closed() whether or not the scan is completed successfully // Eagerly creating scanners so that we have the ref counting ticking on the newly created // store files. In case of stream scanners this eager creation does not induce performance // penalty because in scans (that uses stream scanners) the next() call is bound to happen. List<KeyValueScanner> scanners = store.getScanners(sfs, cacheBlocks, get, usePread, isCompaction, matcher, scan.getStartRow(), scan.getStopRow(), this.readPt, false); flushedstoreFileScanners.addAll(scanners); if (!CollectionUtils.isEmpty(memStoreScanners)) { clearAndClose(memStoreScannersAfterFlush); memStoreScannersAfterFlush.addAll(memStoreScanners); } } finally { flushLock.unlock(); } // Let the next() call handle re-creating and seeking }
model.setStartRow(scan.getStartRow()); model.setEndRow(scan.getStopRow()); Map<byte [], NavigableSet<byte []>> families = scan.getFamilyMap(); if (families != null) { model.setMaxVersions(maxVersions); Filter filter = scan.getFilter(); if (filter != null) { model.setFilter(stringifyFilter(filter));
public static void assertEmptyScanKey(Scan scan) { assertNull(scan.getFilter()); assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, scan.getStartRow()); assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, scan.getStopRow()); assertEquals(null,scan.getFilter()); }
table.coprocessorService(AggregateService.class, scan.getStartRow(), scan.getStopRow(), new Batch.Call<AggregateService, List<S>>() { @Override
byte[] startRow = scan.getStartRow(); if (startRow != null && startRow.length > 0) { scanBuilder.setStartRow(ByteStringer.wrap(startRow)); byte[] stopRow = scan.getStopRow(); if (stopRow != null && stopRow.length > 0) { scanBuilder.setStopRow(ByteStringer.wrap(stopRow)); scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
public static void assertEmptyScanKey(Scan scan) { assertNull(scan.getFilter()); assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, scan.getStartRow()); assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, scan.getStopRow()); assertEquals(null,scan.getFilter()); }
/** * Performs a scan of META table for given table. * @param metaTable * @param startRow Where to start the scan * @param stopRow Where to stop the scan * @param type scanned part of meta * @param maxRows maximum rows to return * @param visitor Visitor invoked against each row */ private static CompletableFuture<Void> scanMeta(AsyncTable<AdvancedScanResultConsumer> metaTable, Optional<byte[]> startRow, Optional<byte[]> stopRow, QueryType type, int maxRows, final Visitor visitor) { int rowUpperLimit = maxRows > 0 ? maxRows : Integer.MAX_VALUE; Scan scan = getMetaScan(metaTable, rowUpperLimit); for (byte[] family : type.getFamilies()) { scan.addFamily(family); } startRow.ifPresent(scan::withStartRow); stopRow.ifPresent(scan::withStopRow); if (LOG.isDebugEnabled()) { LOG.debug("Scanning META" + " starting at row=" + Bytes.toStringBinary(scan.getStartRow()) + " stopping at row=" + Bytes.toStringBinary(scan.getStopRow()) + " for max=" + rowUpperLimit + " with caching=" + scan.getCaching()); } CompletableFuture<Void> future = new CompletableFuture<Void>(); metaTable.scan(scan, new MetaTableScanResultConsumer(rowUpperLimit, visitor, future)); return future; }
byte[] startRow = scan.getStartRow(); if (startRow != null && startRow.length > 0) { scanBuilder.setStartRow(UnsafeByteOperations.unsafeWrap(startRow)); byte[] stopRow = scan.getStopRow(); if (stopRow != null && stopRow.length > 0) { scanBuilder.setStopRow(UnsafeByteOperations.unsafeWrap(stopRow)); scanBuilder.setFilter(ProtobufUtil.toFilter(scan.getFilter()));
@Test public void testScanCopyConstructor() throws Exception { Scan scan = new Scan(); assertEquals(scan.getFamilies()[0], scanCopy.getFamilies()[0]); assertEquals(scan.getFamilyMap(), scanCopy.getFamilyMap()); assertEquals(scan.getFilter(), scanCopy.getFilter()); assertEquals(scan.getId(), scanCopy.getId()); assertEquals(scan.getIsolationLevel(), scanCopy.getIsolationLevel()); assertEquals(scan.getReplicaId(), scanCopy.getReplicaId()); assertEquals(scan.getRowOffsetPerColumnFamily(), scanCopy.getRowOffsetPerColumnFamily()); assertEquals(scan.getStartRow(), scanCopy.getStartRow()); assertEquals(scan.getStopRow(), scanCopy.getStopRow()); assertEquals(scan.getTimeRange(), scanCopy.getTimeRange());
private boolean isValidRegion(RegionInfo hri) { // An offline split parent region should be excluded. if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) { return false; } return PrivateCellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey()); }