private static void setRowPrefixFilter(Scan scan, String rowPrefixes) { if (rowPrefixes != null && !rowPrefixes.isEmpty()) { String[] rowPrefixArray = rowPrefixes.split(","); Arrays.sort(rowPrefixArray); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); for (String prefix : rowPrefixArray) { Filter filter = new PrefixFilter(Bytes.toBytes(prefix)); filterList.addFilter(filter); } scan.setFilter(filterList); byte[] startPrefixRow = Bytes.toBytes(rowPrefixArray[0]); byte[] lastPrefixRow = Bytes.toBytes(rowPrefixArray[rowPrefixArray.length -1]); setStartAndStopRows(scan, startPrefixRow, lastPrefixRow); } }
@Test public void testPrefixFilter() throws Exception { // Grab rows from group one (half of total) long expectedRows = this.numRows / 2; long expectedKeys = this.colsPerRow; Scan s = new Scan(); s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); verifyScan(s, expectedRows, expectedKeys); }
private void runScanner(Table hTable, int expectedSize, Filter filter) throws IOException { String cf = "f"; Scan scan = new Scan(); scan.addFamily(cf.getBytes()); scan.setFilter(filter); List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(table.getBytes()); HRegion first = regions.get(0); first.getScanner(scan); RegionScanner scanner = first.getScanner(scan); List<Cell> results = new ArrayList<>(); // Result result; long timeBeforeScan = System.currentTimeMillis(); int found = 0; while (scanner.next(results)) { found += results.size(); results.clear(); } found += results.size(); long scanTime = System.currentTimeMillis() - timeBeforeScan; scanner.close(); LOG.info("\nscan time = " + scanTime + "ms"); LOG.info("found " + found + " results\n"); assertEquals(expectedSize, found); }
@Override protected void initialize(JobContext job) throws IOException { Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create( job.getConfiguration())); TableName tableName = TableName.valueOf("exampleTable"); // mandatory initializeTable(connection, tableName); byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"), Bytes.toBytes("columnB") }; //optional Scan scan = new Scan(); for (byte[] family : inputColumns) { scan.addFamily(family); } Filter exampleFilter = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*")); scan.setFilter(exampleFilter); setScan(scan); }
private int cellsCount(Table table, Filter filter) throws IOException { Scan scan = new Scan().setFilter(filter).readAllVersions(); try (ResultScanner scanner = table.getScanner(scan)) { List<Cell> results = new ArrayList<>(); Result result; while ((result = scanner.next()) != null) { result.listCells().forEach(results::add); } return results.size(); } }
@Override public void configure(JobConf job) { try { Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(job)); TableName tableName = TableName.valueOf("exampleJobConfigurableTable"); // mandatory initializeTable(connection, tableName); byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"), Bytes.toBytes("columnB") }; //optional Scan scan = new Scan(); for (byte[] family : inputColumns) { scan.addFamily(family); } Filter exampleFilter = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*")); scan.setFilter(exampleFilter); setScan(scan); } catch (IOException exception) { throw new RuntimeException("Failed to initialize.", exception); } } }
static private void removeTablePermissions(TableName tableName, byte[] column, Table table, boolean closeTable) throws IOException { Scan scan = new Scan(); scan.addFamily(ACL_LIST_FAMILY); String columnName = Bytes.toString(column); scan.setFilter(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator( String.format("(%s%s%s)|(%s%s)$", ACL_KEY_DELIMITER, columnName, ACL_KEY_DELIMITER, ACL_KEY_DELIMITER, columnName)))); Set<byte[]> qualifierSet = new TreeSet<>(Bytes.BYTES_COMPARATOR); ResultScanner scanner = null; try { scanner = table.getScanner(scan); for (Result res : scanner) { for (byte[] q : res.getFamilyMap(ACL_LIST_FAMILY).navigableKeySet()) { qualifierSet.add(q); } } if (qualifierSet.size() > 0) { Delete d = new Delete(tableName.getName()); for (byte[] qualifier : qualifierSet) { d.addColumns(ACL_LIST_FAMILY, qualifier); } table.delete(d); } } finally { if (scanner != null) scanner.close(); if (closeTable) table.close(); } }
@Override public Map<String, Object> count(String task) { MappingConfig config = hbaseMapping.get(task); String hbaseTable = config.getHbaseMapping().getHbaseTable(); long rowCount = 0L; try { HTable table = (HTable) hbaseTemplate.getConnection().getTable(TableName.valueOf(hbaseTable)); Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); ResultScanner resultScanner = table.getScanner(scan); for (Result result : resultScanner) { rowCount += result.size(); } } catch (IOException e) { logger.error(e.getMessage(), e); } Map<String, Object> res = new LinkedHashMap<>(); res.put("hbaseTable", hbaseTable); res.put("count", rowCount); return res; }
@Test public void testPrefixFilterWithReverseScan() throws Exception { // Grab rows from group one (half of total) long expectedRows = this.numRows / 2; long expectedKeys = this.colsPerRow; Scan s = new Scan(); s.setReversed(true); s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); verifyScan(s, expectedRows, expectedKeys); }
/** * Test the filter by adding 2 columns of family A in the scan. (OK) */ @Test public void scanWithSpecificQualifiers3() throws IOException { /* When */ Scan scan = new Scan(); scan.addColumn(FAMILY_A, QUALIFIER_FOO); scan.addColumn(FAMILY_A, QUALIFIER_BAR); scan.setFilter(scanFilter); verify(scan); }
private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) throws IOException { // Defaults FilterList.Operator.MUST_PASS_ALL. FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); // Only return rows where this column value exists in the row. SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return r.getScanner(scan); }
/** * Count the number of keyvalues in the specified table with the given filter * @param table the table to scan * @return the number of keyvalues found * @throws IOException */ private int getCount(Table table, Filter filter) throws IOException { Scan scan = new Scan(); scan.setFilter(filter); ResultScanner results = table.getScanner(scan); int count = 0; for (Result res : results) { count += res.size(); } results.close(); return count; }
/** * Uses the TimestampFilter on a Scan to request a specified list of * versions for the rows from startRowIdx to endRowIdx (both inclusive). */ private Result[] scanNVersions(Table ht, byte[] cf, int startRowIdx, int endRowIdx, List<Long> versions) throws IOException { byte startRow[] = Bytes.toBytes("row:" + startRowIdx); byte endRow[] = Bytes.toBytes("row:" + endRowIdx + 1); // exclusive Filter filter = new TimestampsFilter(versions); Scan scan = new Scan(startRow, endRow); scan.setFilter(filter); scan.setMaxVersions(); ResultScanner scanner = ht.getScanner(scan); return scanner.next(endRowIdx - startRowIdx + 1); }
/** * Test the filter by adding 1 column of family A (the one used in the filter) and 1 column of * family B in the scan. (OK) */ @Test public void scanWithSpecificQualifiers2() throws IOException { /* When */ Scan scan = new Scan(); scan.addColumn(FAMILY_A, QUALIFIER_FOO); scan.addColumn(FAMILY_B, QUALIFIER_BAR); scan.setFilter(scanFilter); verify(scan); }
@Override public void configure(JobConf job) { try { Connection connection = ConnectionFactory.createConnection(job); Table exampleTable = connection.getTable(TableName.valueOf(("exampleDeprecatedTable"))); // mandatory initializeTable(connection, exampleTable.getName()); byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"), Bytes.toBytes("columnB") }; // optional Scan scan = new Scan(); for (byte[] family : inputColumns) { scan.addFamily(family); } Filter exampleFilter = new RowFilter(CompareOperator.EQUAL, new RegexStringComparator("aa.*")); scan.setFilter(exampleFilter); setScan(scan); } catch (IOException exception) { throw new RuntimeException("Failed to configure for job.", exception); } }