Refine search
@Test public void testPrefixFilterWithReverseScan() throws Exception { // Grab rows from group one (half of total) long expectedRows = this.numRows / 2; long expectedKeys = this.colsPerRow; Scan s = new Scan(); s.setReversed(true); s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne"))); verifyScan(s, expectedRows, expectedKeys); }
case ColumnPrefixFilter: byte[] src = ((ColumnPrefixFilter)filter).getPrefix(); this.value = Bytes.toString(Base64.getEncoder().encode(src)); break; case ColumnRangeFilter: ColumnRangeFilter crf = (ColumnRangeFilter)filter; this.minColumn = Bytes.toString(Base64.getEncoder().encode(crf.getMinColumn())); this.minColumnInclusive = crf.getMinColumnInclusive(); this.maxColumn = Bytes.toString(Base64.getEncoder().encode(crf.getMaxColumn())); this.maxColumnInclusive = crf.getMaxColumnInclusive(); break; } break; case FilterList: this.op = ((FilterList)filter).getOperator().toString(); this.filters = new ArrayList<>(); for (Filter child: ((FilterList)filter).getFilters()) { this.filters.add(new FilterModel(child)); case PrefixFilter: this.value = Bytes.toString(Base64.getEncoder().encode( ((PrefixFilter)filter).getPrefix())); break; case FamilyFilter:
@Override public boolean filterRowKey(Cell firstRowCell) { if (firstRowCell == null || this.prefix == null) return true; if (filterAllRemaining()) return true; int length = firstRowCell.getRowLength(); if (length < prefix.length) return true; // if they are equal, return false => pass row // else return true, filter row // if we are passed the prefix, set flag int cmp; if (firstRowCell instanceof ByteBufferExtendedCell) { cmp = ByteBufferUtils.compareTo(((ByteBufferExtendedCell) firstRowCell).getRowByteBuffer(), ((ByteBufferExtendedCell) firstRowCell).getRowPosition(), this.prefix.length, this.prefix, 0, this.prefix.length); } else { cmp = Bytes.compareTo(firstRowCell.getRowArray(), firstRowCell.getRowOffset(), this.prefix.length, this.prefix, 0, this.prefix.length); } if ((!isReversed() && cmp > 0) || (isReversed() && cmp < 0)) { passedPrefix = true; } filterRow = (cmp != 0); return filterRow; }
@Test public void testPrefixFilter() throws Exception { // null prefix PrefixFilter prefixFilter = new PrefixFilter(null); assertTrue(prefixFilter.areSerializedFieldsEqual( ProtobufUtil.toFilter(ProtobufUtil.toFilter(prefixFilter)))); // non-null prefix prefixFilter = new PrefixFilter(Bytes.toBytes("abc")); assertTrue(prefixFilter.areSerializedFieldsEqual( ProtobufUtil.toFilter(ProtobufUtil.toFilter(prefixFilter)))); }
@Test public void testReversedFilterListWithOR() throws IOException { byte[] r22 = Bytes.toBytes("Row22"); byte[] r2 = Bytes.toBytes("Row2"); byte[] r1 = Bytes.toBytes("Row1"); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.setReversed(true); PrefixFilter prefixFilter = new PrefixFilter(r2); prefixFilter.setReversed(true); filterList.addFilter(prefixFilter); filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r22)); assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r22, r22, r22))); AlwaysNextColFilter alwaysNextColFilter = new AlwaysNextColFilter(); alwaysNextColFilter.setReversed(true); prefixFilter = new PrefixFilter(r2); prefixFilter.setReversed(true); filterList.addFilter(alwaysNextColFilter); filterList.addFilter(prefixFilter);
private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) throws IOException { // Defaults FilterList.Operator.MUST_PASS_ALL. FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); // Only return rows where this column value exists in the row. SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return r.getScanner(scan); }
try { Filter prefixFilter = null; Scan tableScan = new Scan(); if (scanSpec.indexOf('*') > 0) { String prefix = scanSpec.substring(0, scanSpec.indexOf('*')); byte[] prefixBytes = Bytes.toBytes(prefix); prefixFilter = new PrefixFilter(Bytes.toBytes(prefix)); if (startRow.isEmpty()) { tableScan.setStartRow(prefixBytes); tableScan.setBatch(batchSize); tableScan.setMaxVersions(maxVersions); tableScan.setTimeRange(startTime, endTime); if (!startRow.isEmpty()) { tableScan.setStartRow(Bytes.toBytes(startRow)); tableScan.setStopRow(Bytes.toBytes(endRow)); FilterList filterList = new FilterList(); if (StringUtils.isNotEmpty(paramFilter)) { ParseFilter pf = new ParseFilter(); Filter parsedParamFilter = pf.parseFilterString(paramFilter); if (parsedParamFilter != null) { filterList.addFilter(parsedParamFilter); filterList.addFilter(prefixFilter);
/** * creates a scan for flow data * @param rowPrefix - start row prefix * @param limit - limit on scanned results * @param version - version to match * @return Scan */ private Scan createFlowScan(byte[] rowPrefix, int limit, String version) { Scan scan = new Scan(); scan.setStartRow(rowPrefix); // using a large scanner caching value with a small limit can mean we scan a // lot more data than necessary, so lower the caching for low limits scan.setCaching(Math.min(limit, defaultScannerCaching)); // require that all rows match the prefix we're looking for Filter prefixFilter = new WhileMatchFilter(new PrefixFilter(rowPrefix)); // if version is passed, restrict the rows returned to that version if (version != null && version.length() > 0) { FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL); filters.addFilter(prefixFilter); filters.addFilter(new SingleColumnValueFilter(Constants.INFO_FAM_BYTES, Constants.VERSION_COLUMN_BYTES, CompareFilter.CompareOp.EQUAL, Bytes.toBytes(version))); scan.setFilter(filters); } else { scan.setFilter(prefixFilter); } return scan; }
byte[] startRow = null; if (StringUtils.isNotBlank(user)) { startRow = ByteUtil.join(Constants.SEP_BYTES, Bytes.toBytes(cluster), Bytes.toBytes(user)); } else { startRow = ByteUtil.join(Constants.SEP_BYTES, Bytes.toBytes(cluster)); "Reading app version rows start at " + Bytes.toStringBinary(startRow)); Scan scan = new Scan(); scan.setStartRow(startRow); FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL); filters.addFilter(new WhileMatchFilter(new PrefixFilter(startRow))); scan.setFilter(filters);
@Test public void testFilterListWithPrefixFilter() throws IOException { byte[] family = Bytes.toBytes("f1"); byte[] qualifier = Bytes.toBytes("q1"); HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); htd.addFamily(new HColumnDescriptor(family)); Put p = new Put(Bytes.toBytes((char)('a'+i) + "row")); p.setDurability(Durability.SKIP_WAL); p.addColumn(family, qualifier, Bytes.toBytes(String.valueOf(111 + i))); PrefixFilter pf = new PrefixFilter(new byte[] {'b'}) ; family, qualifier, CompareOperator.EQUAL, Bytes.toBytes("113")); FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, pf, scvf); Scan s1 = new Scan(); s1.setFilter(filterList); InternalScanner scanner = testRegion.getScanner(s1); List<Cell> results = new ArrayList<>();
int maxVersions = 2; Scan scan = new Scan(startRow, stopRow); scan.addColumn(fam, qf1); scan.setTimeRange(ts, ts+1); scan.setMaxVersions(maxVersions); Scan desScan = ProtobufUtil.toScan(scanProto); assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow())); assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow())); assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks()); Set<byte[]> set = null; scan = new Scan(startRow); final String name = "testScan"; byte [] prefix = Bytes.toBytes(name); scan.setFilter(new PrefixFilter(prefix)); scanProto = ProtobufUtil.toScan(scan); desScan = ProtobufUtil.toScan(scanProto);
byte[] rowPrefix = Bytes.toBytes((cluster + Constants.SEP + user + Constants.SEP + appId + Constants.SEP)); byte[] scanStartRow; Bytes.add(rowPrefix, Bytes.toBytes(endRunId), Constants.SEP_BYTES); } else { scanStartRow = rowPrefix; Scan scan = new Scan(); scan.setStartRow(scanStartRow); FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL); byte[] scanEndRow = Bytes.add(rowPrefix, Bytes.toBytes(startRunId), Constants.ZERO_SINGLE_BYTE); scan.setStopRow(scanEndRow); } else { filters.addFilter(new WhileMatchFilter(new PrefixFilter(rowPrefix))); filters.addFilter(new SingleColumnValueFilter(Constants.INFO_FAM_BYTES, Constants.VERSION_COLUMN_BYTES, CompareFilter.CompareOp.EQUAL, Bytes.toBytes(version)));
private static void setRowPrefixFilter(Scan scan, String rowPrefixes) { if (rowPrefixes != null && !rowPrefixes.isEmpty()) { String[] rowPrefixArray = rowPrefixes.split(","); Arrays.sort(rowPrefixArray); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); for (String prefix : rowPrefixArray) { Filter filter = new PrefixFilter(Bytes.toBytes(prefix)); filterList.addFilter(filter); } scan.setFilter(filterList); byte[] startPrefixRow = Bytes.toBytes(rowPrefixArray[0]); byte[] lastPrefixRow = Bytes.toBytes(rowPrefixArray[rowPrefixArray.length -1]); setStartAndStopRows(scan, startPrefixRow, lastPrefixRow); } }
private Filter getMPALLFilter() { List<Filter> filters = new ArrayList<>(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); return filterMPALL; }
/** * Returns a Scan instance to retrieve all the task rows for a given job from * the job_history_task table. * @param jobKey the job key to match for all task rows * @return a {@code Scan} instance for the job_history_task table */ private Scan getTaskScan(JobKey jobKey) { byte[] startKey = Bytes.add(jobKeyConv.toBytes(jobKey), Constants.SEP_BYTES); Scan scan = new Scan(); scan.setStartRow(startKey); // only return tasks for this job scan.setFilter(new WhileMatchFilter(new PrefixFilter(startKey))); // expect a lot of tasks on average scan.setCaching(500); return scan; }
HConstants.DEFAULT_HBASE_META_SCANNER_CACHING)); final Scan scan = new Scan(startRow); if (LOG.isDebugEnabled()) { LOG.debug("Scanning " + metaTableString + " starting at row=" + Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " rows "); scan.addFamily(FConstants.CATALOG_FAMILY); scan.setCaching(rows); if (tableName == null || tableName.length == 0) { FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(tableName)); scan.setFilter(allFilters); break; if (Bytes.startsWith(r.getRow(), FConstants.TABLEROW_PREFIX)) { continue;
private void testScanWithFilters(Connection connection, String tableName) throws IOException { createTable(thriftAdmin, tableName); try (Table table = connection.getTable(TableName.valueOf(tableName))){ FilterList filterList = new FilterList(); PrefixFilter prefixFilter = new PrefixFilter(Bytes.toBytes("testrow")); ColumnValueFilter columnValueFilter = new ColumnValueFilter(FAMILYA, QUALIFIER_1, CompareOperator.EQUAL, VALUE_1); filterList.addFilter(prefixFilter); filterList.addFilter(columnValueFilter); Scan scan = new Scan(); scan.setMaxVersions(2); ResultScanner scanner = table.getScanner(scan); Iterator<Result> iterator = scanner.iterator(); assertTrue(iterator.hasNext()); int counter = 0; while (iterator.hasNext()) { Result result = iterator.next(); counter += result.size(); } assertEquals(2, counter); } }
try { table = getTable(tableName); Scan scan = new Scan(getBytes(startAndPrefix)); addAttributes(scan, attributes); Filter f = new WhileMatchFilter( new PrefixFilter(getBytes(startAndPrefix))); scan.setFilter(f); if (columns != null && !columns.isEmpty()) { for(ByteBuffer column : columns) { byte [][] famQf = CellUtil.parseColumn(getBytes(column)); if(famQf.length == 1) { scan.addFamily(famQf[0]); } else { scan.addColumn(famQf[0], famQf[1]);
/** * When we do a "MUST_PASS_ONE" (a logical 'OR') of the above two filters * we expect to get the same result as the 'prefix' only result. * @throws Exception */ @Test public void testFilterListTwoFiltersMustPassOne() throws Exception { byte[] r1 = Bytes.toBytes("Row1"); byte[] r11 = Bytes.toBytes("Row11"); byte[] r2 = Bytes.toBytes("Row2"); FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new PrefixFilter(r1)); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1)); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1))); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11))); flist.reset(); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2)); assertEquals(ReturnCode.SKIP, flist.filterCell(new KeyValue(r2, r2, r2))); flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new AlwaysNextColFilter()); flist.addFilter(new PrefixFilter(r1)); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1)); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1))); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11))); flist.reset(); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2)); assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2))); }
/** * Test serialization * @throws Exception */ @Test public void testSerialization() throws Exception { List<Filter> filters = new ArrayList<>(); filters.add(new PageFilter(MAX_PAGES)); filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy")))); Filter filterMPALL = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); // Decompose filterMPALL to bytes. byte[] buffer = filterMPALL.toByteArray(); // Recompose filterMPALL. FilterList newFilter = FilterList.parseFrom(buffer); // Run tests mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE()))); mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter()))); orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter()))); }