Refine search
private static void setRowPrefixFilter(Scan scan, String rowPrefixes) { if (rowPrefixes != null && !rowPrefixes.isEmpty()) { String[] rowPrefixArray = rowPrefixes.split(","); Arrays.sort(rowPrefixArray); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); for (String prefix : rowPrefixArray) { Filter filter = new PrefixFilter(Bytes.toBytes(prefix)); filterList.addFilter(filter); } scan.setFilter(filterList); byte[] startPrefixRow = Bytes.toBytes(rowPrefixArray[0]); byte[] lastPrefixRow = Bytes.toBytes(rowPrefixArray[rowPrefixArray.length -1]); setStartAndStopRows(scan, startPrefixRow, lastPrefixRow); } }
private FilterList generateTimeFilterList(VisitFilter visitFilter) { FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); if (visitFilter.lastModStart >= 0) { // NOTE: Negative value does not work in its binary form SingleColumnValueFilter timeStartFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.GREATER_OR_EQUAL, Bytes.toBytes(visitFilter.lastModStart)); filterList.addFilter(timeStartFilter); } if (visitFilter.lastModEndExclusive != Long.MAX_VALUE) { SingleColumnValueFilter timeEndFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.LESS, Bytes.toBytes(visitFilter.lastModEndExclusive)); filterList.addFilter(timeEndFilter); } return filterList.getFilters().size() == 0 ? null : filterList; }
public static void applyFuzzyFilter(Scan scan, List<org.apache.kylin.common.util.Pair<byte[], byte[]>> fuzzyKeys) { if (fuzzyKeys != null && fuzzyKeys.size() > 0) { FuzzyRowFilter rowFilter = new FuzzyRowFilter(convertToHBasePair(fuzzyKeys)); Filter filter = scan.getFilter(); if (filter != null) { // may have existed InclusiveStopFilter, see buildScan FilterList filterList = new FilterList(); filterList.addFilter(filter); filterList.addFilter(rowFilter); scan.setFilter(filterList); } else { scan.setFilter(rowFilter); } } }
FilterList filterList = new FilterList(); if (StringUtils.isNotEmpty(paramFilter)) { ParseFilter pf = new ParseFilter(); Filter parsedParamFilter = pf.parseFilterString(paramFilter); if (parsedParamFilter != null) { filterList.addFilter(parsedParamFilter); filterList.addFilter(prefixFilter);
FilterList nsFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL); nsFilters.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); nsFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator( getSettingsQualifierRegexForUserNamespace(filter.getNamespaceFilter()), 0))); userFilters.addFilter(nsFilters); hasFilter = true; tableFilters.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); tableFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator( getSettingsQualifierRegexForUserTable(filter.getTableFilter()), 0))); userFilters.addFilter(tableFilters); hasFilter = true; userFilters.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0))); filterList.addFilter(userFilters); } else if (StringUtils.isNotEmpty(filter.getTableFilter())) { filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getTableRowKeyRegex(filter.getTableFilter()), 0))); } else if (StringUtils.isNotEmpty(filter.getNamespaceFilter())) { filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0))); } else if (StringUtils.isNotEmpty(filter.getRegionServerFilter())) { filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(
@Test public void testAddFilter() throws Exception { Filter filter1 = new FirstKeyOnlyFilter(); Filter filter2 = new FirstKeyOnlyFilter(); FilterList filterList = new FilterList(filter1, filter2); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Arrays.asList(filter1, filter2)); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2)); filterList.addFilter(new FirstKeyOnlyFilter()); filterList.setReversed(false); FirstKeyOnlyFilter f = new FirstKeyOnlyFilter(); f.setReversed(true); try { filterList.addFilter(f); fail("The IllegalArgumentException should be thrown because the added filter is reversed"); } catch (IllegalArgumentException e) { } }
private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) throws IOException { // Defaults FilterList.Operator.MUST_PASS_ALL. FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); // Only return rows where this column value exists in the row. SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return r.getScanner(scan); }
private ResultScanner buildScanner(String keyPrefix, String value, Table ht) throws IOException { // OurFilterList allFilters = new OurFilterList(); FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes .toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); // allFilters.addFilter(new // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return ht.getScanner(scan); }
@Test public void testMultiRowRangeWithFilterListAndOperator() throws IOException { tableName = TableName.valueOf(name.getMethodName()); Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); generateRows(numRows, ht, family, qf, value); Scan scan = new Scan(); scan.setMaxVersions(); List<RowRange> ranges1 = new ArrayList<>(); ranges1.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(20), false)); ranges1.add(new RowRange(Bytes.toBytes(30), true, Bytes.toBytes(40), false)); ranges1.add(new RowRange(Bytes.toBytes(60), true, Bytes.toBytes(70), false)); MultiRowRangeFilter filter1 = new MultiRowRangeFilter(ranges1); List<RowRange> ranges2 = new ArrayList<>(); ranges2.add(new RowRange(Bytes.toBytes(20), true, Bytes.toBytes(40), false)); ranges2.add(new RowRange(Bytes.toBytes(80), true, Bytes.toBytes(90), false)); MultiRowRangeFilter filter2 = new MultiRowRangeFilter(ranges2); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); filterList.addFilter(filter1); filterList.addFilter(filter2); scan.setFilter(filterList); int resultsSize = getResultsSize(ht, scan); LOG.info("found " + resultsSize + " results"); List<Cell> results1 = getScanResult(Bytes.toBytes(30), Bytes.toBytes(40), ht); assertEquals(results1.size(), resultsSize); ht.close(); }
@Test public void testMultiRowRangeWithFilterListOrOperator() throws IOException { tableName = TableName.valueOf(name.getMethodName()); Table ht = TEST_UTIL.createTable(tableName, family, Integer.MAX_VALUE); generateRows(numRows, ht, family, qf, value); Scan scan = new Scan(); scan.setMaxVersions(); List<RowRange> ranges1 = new ArrayList<>(); ranges1.add(new RowRange(Bytes.toBytes(30), true, Bytes.toBytes(40), false)); ranges1.add(new RowRange(Bytes.toBytes(10), true, Bytes.toBytes(20), false)); ranges1.add(new RowRange(Bytes.toBytes(60), true, Bytes.toBytes(70), false)); MultiRowRangeFilter filter1 = new MultiRowRangeFilter(ranges1); List<RowRange> ranges2 = new ArrayList<>(); ranges2.add(new RowRange(Bytes.toBytes(20), true, Bytes.toBytes(40), false)); ranges2.add(new RowRange(Bytes.toBytes(80), true, Bytes.toBytes(90), false)); MultiRowRangeFilter filter2 = new MultiRowRangeFilter(ranges2); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(filter1); filterList.addFilter(filter2); scan.setFilter(filterList); int resultsSize = getResultsSize(ht, scan); LOG.info("found " + resultsSize + " results"); List<Cell> results1 = getScanResult(Bytes.toBytes(10), Bytes.toBytes(40), ht); List<Cell> results2 = getScanResult(Bytes.toBytes(60), Bytes.toBytes(70), ht); List<Cell> results3 = getScanResult(Bytes.toBytes(80), Bytes.toBytes(90), ht); assertEquals(results1.size() + results2.size() + results3.size(),resultsSize); ht.close(); }
@Test public void testReversedFilterListWithMockSeekHintFilter() throws IOException { KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, Bytes.toBytes("value")); KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, Bytes.toBytes("value")); KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1, Bytes.toBytes("value")); Filter filter1 = new MockSeekHintFilter(kv1); filter1.setReversed(true); Filter filter2 = new MockSeekHintFilter(kv2); filter2.setReversed(true); Filter filter3 = new MockSeekHintFilter(kv3); filter3.setReversed(true); FilterList filterList = new FilterList(Operator.MUST_PASS_ONE); filterList.setReversed(true); filterList.addFilter(filter1); filterList.addFilter(filter2); filterList.addFilter(filter3); Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1)); Assert.assertEquals(kv3, filterList.getNextCellHint(kv1)); filterList = new FilterList(Operator.MUST_PASS_ALL); filterList.setReversed(true); filterList.addFilter(filter1); filterList.addFilter(filter2); filterList.addFilter(filter3); Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1)); Assert.assertEquals(kv1, filterList.getNextCellHint(kv1)); }
private void testScanWithFilters(Connection connection, String tableName) throws IOException { createTable(thriftAdmin, tableName); try (Table table = connection.getTable(TableName.valueOf(tableName))){ FilterList filterList = new FilterList(); PrefixFilter prefixFilter = new PrefixFilter(Bytes.toBytes("testrow")); ColumnValueFilter columnValueFilter = new ColumnValueFilter(FAMILYA, QUALIFIER_1, CompareOperator.EQUAL, VALUE_1); filterList.addFilter(prefixFilter); filterList.addFilter(columnValueFilter); Scan scan = new Scan(); scan.setMaxVersions(2); ResultScanner scanner = table.getScanner(scan); Iterator<Result> iterator = scanner.iterator(); assertTrue(iterator.hasNext()); int counter = 0; while (iterator.hasNext()) { Result result = iterator.next(); counter += result.size(); } assertEquals(2, counter); } }
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(filter1); filterList.addFilter(filter2); scan.setFilter(filterList); int resultsSize = getResultsSize(ht, scan);
@Test public void testFiltersWithOR() throws Exception { TableName tn = TableName.valueOf(name.getMethodName()); Table table = TEST_UTIL.createTable(tn, new String[] { "cf1", "cf2" }); byte[] CF1 = Bytes.toBytes("cf1"); byte[] CF2 = Bytes.toBytes("cf2"); Put put1 = new Put(Bytes.toBytes("0")); put1.addColumn(CF1, Bytes.toBytes("col_a"), Bytes.toBytes(0)); table.put(put1); Put put2 = new Put(Bytes.toBytes("0")); put2.addColumn(CF2, Bytes.toBytes("col_b"), Bytes.toBytes(0)); table.put(put2); FamilyFilter filterCF1 = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(CF1)); FamilyFilter filterCF2 = new FamilyFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator(CF2)); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(filterCF1); filterList.addFilter(filterCF2); Scan scan = new Scan(); scan.setFilter(filterList); ResultScanner scanner = table.getScanner(scan); LOG.info("Filter list: " + filterList); for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { Assert.assertEquals(2, rr.size()); } }
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); filterList.addFilter(filter); scan.setFilter(filterList);
protected Scan constructScan(byte[] valuePrefix) throws IOException { FilterList list = new FilterList(); Filter filter = new SingleColumnValueFilter(FAMILY_ZERO, COLUMN_ZERO, CompareOperator.EQUAL, new BinaryComparator(valuePrefix)); list.addFilter(filter); if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) .setScanMetricsEnabled(true); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(FAMILY_ZERO, qualifier); } } else { scan.addFamily(FAMILY_ZERO); } scan.setFilter(list); return scan; } }
/** * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters * we expect to get the same result as the inclusive stop result. * @throws Exception */ @Test public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception { byte[] r1 = Bytes.toBytes("Row1"); byte[] r11 = Bytes.toBytes("Row11"); byte[] r2 = Bytes.toBytes("Row2"); FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE); flist.addFilter(new AlwaysNextColFilter()); flist.addFilter(new InclusiveStopFilter(r1)); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1)); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1))); assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11))); flist.reset(); flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2)); assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2))); }
FilterList subFilterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); Filter subFilter1 = new RowFilter(CompareOperator.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes("row4"))); subFilterList.addFilter(subFilter1); Filter subFilter2 = new SingleColumnValueFilter(FAMILIES[0], columnStatus, CompareOperator.EQUAL, Bytes.toBytes(0)); subFilterList.addFilter(subFilter2); s1 = new Scan(); s1.setFilter(subFilterList); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(rowFilter); filterList.addFilter(subFilterList); s1 = new Scan(); s1.setFilter(filterList); filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE); filterList.addFilter(subFilterList); filterList.addFilter(rowFilter); s1 = new Scan(); s1.setFilter(filterList);
/** * Sets the filter. * * @param filter * the new filter */ public void setFilter(Filter filter) { if (this.filter == null) { this.filter = new FilterList(); } if (filter != null) { this.filter.addFilter(filter); } }
/** * Set filter to data handler. * * @param filter * hbase filter. */ public void setFilter(Filter filter) { if (this.filter == null) { this.filter = new FilterList(); } if (filter != null) { this.filter.addFilter(filter); } }