private Filter basicFilterNew() { return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOperator.GREATER_OR_EQUAL, VAL_2); }
public Filter getScanFilter() { SingleColumnValueFilter scf = new SingleColumnValueFilter(ESSENTIAL_CF, FILTER_COLUMN, CompareOperator.EQUAL, Bytes.toBytes(ACCEPTED_VALUE)); scf.setFilterIfMissing(true); return scf; } }
private FilterList generateTimeFilterList(VisitFilter visitFilter) { FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); if (visitFilter.lastModStart >= 0) { // NOTE: Negative value does not work in its binary form SingleColumnValueFilter timeStartFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.GREATER_OR_EQUAL, Bytes.toBytes(visitFilter.lastModStart)); filterList.addFilter(timeStartFilter); } if (visitFilter.lastModEndExclusive != Long.MAX_VALUE) { SingleColumnValueFilter timeEndFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS, CompareFilter.CompareOp.LESS, Bytes.toBytes(visitFilter.lastModEndExclusive)); filterList.addFilter(timeEndFilter); } return filterList.getFilters().size() == 0 ? null : filterList; }
public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments) { Preconditions.checkArgument(filterArguments.size() == 4 || filterArguments.size() == 6, "Expected 4 or 6 but got: %s", filterArguments.size()); byte [] family = ParseFilter.removeQuotesFromByteArray(filterArguments.get(0)); byte [] qualifier = ParseFilter.removeQuotesFromByteArray(filterArguments.get(1)); CompareOperator op = ParseFilter.createCompareOperator(filterArguments.get(2)); org.apache.hadoop.hbase.filter.ByteArrayComparable comparator = ParseFilter.createComparator( ParseFilter.removeQuotesFromByteArray(filterArguments.get(3))); if (comparator instanceof RegexStringComparator || comparator instanceof SubstringComparator) { if (op != CompareOperator.EQUAL && op != CompareOperator.NOT_EQUAL) { throw new IllegalArgumentException ("A regexstring comparator and substring comparator " + "can only be used with EQUAL and NOT_EQUAL"); } } SingleColumnValueFilter filter = new SingleColumnValueFilter(family, qualifier, op, comparator); if (filterArguments.size() == 6) { boolean filterIfMissing = ParseFilter.convertByteArrayToBoolean(filterArguments.get(4)); boolean latestVersionOnly = ParseFilter.convertByteArrayToBoolean(filterArguments.get(5)); filter.setFilterIfMissing(filterIfMissing); filter.setLatestVersionOnly(latestVersionOnly); } return filter; }
private Filter substrFilterNew() { return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOperator.EQUAL, new SubstringComparator(QUICK_SUBSTR)); }
private Filter regexFilterNew() { return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOperator.EQUAL, new RegexStringComparator(QUICK_REGEX)); }
private Filter nullFilterNew() { return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOperator.NOT_EQUAL, new NullComparator()); }
private Filter regexFilterNew(Pattern pattern) { return new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOperator.EQUAL, new RegexStringComparator(pattern.pattern(), pattern.flags())); }
protected Scan constructScan(byte[] valuePrefix) throws IOException { Filter filter = new SingleColumnValueFilter( FAMILY_NAME, QUALIFIER_NAME, CompareOperator.EQUAL, new BinaryComparator(valuePrefix) ); Scan scan = new Scan(); scan.addColumn(FAMILY_NAME, QUALIFIER_NAME); scan.setFilter(filter); return scan; } }
/** * Tests the the {@link WhileMatchFilter} works in combination with a * {@link Filter} that uses the {@link Filter#filterCell(Cell)} method. * * See HBASE-2258. * * @throws Exception */ @Test public void testWhileMatchFilterWithFilterCell() throws Exception { Scan s = new Scan(); WhileMatchFilter filter = new WhileMatchFilter( new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOperator.EQUAL, Bytes.toBytes("foo")) ); s.setFilter(filter); InternalScanner scanner = this.region.getScanner(s); while (true) { ArrayList<Cell> values = new ArrayList<>(); boolean isMoreResults = scanner.next(values); assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining()); if (!isMoreResults) { break; } } }
private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) throws IOException { // Defaults FilterList.Operator.MUST_PASS_ALL. FilterList allFilters = new FilterList(); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); // Only return rows where this column value exists in the row. SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return r.getScanner(scan); }
@Test public void testIndexesScanWithOneDeletedRow() throws IOException { byte[] family = Bytes.toBytes("family"); // Setting up region this.region = initHRegion(tableName, method, CONF, family); Put put = new Put(Bytes.toBytes(1L)); put.addColumn(family, qual1, 1L, Bytes.toBytes(1L)); region.put(put); region.flush(true); Delete delete = new Delete(Bytes.toBytes(1L), 1L); region.delete(delete); put = new Put(Bytes.toBytes(2L)); put.addColumn(family, qual1, 2L, Bytes.toBytes(2L)); region.put(put); Scan idxScan = new Scan(); idxScan.addFamily(family); idxScan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.<Filter> asList( new SingleColumnValueFilter(family, qual1, CompareOp.GREATER_OR_EQUAL, new BinaryComparator(Bytes.toBytes(0L))), new SingleColumnValueFilter(family, qual1, CompareOp.LESS_OR_EQUAL, new BinaryComparator(Bytes.toBytes(3L)))))); InternalScanner scanner = region.getScanner(idxScan); List<Cell> res = new ArrayList<>(); while (scanner.next(res)) { // Ignore res value. } assertEquals(1L, res.size()); }
private void runScanner(Table table, boolean slow) throws Exception { long time = System.nanoTime(); Scan scan = new Scan(); scan.addColumn(cf_essential, col_name); scan.addColumn(cf_joined, col_name); SingleColumnValueFilter filter = new SingleColumnValueFilter( cf_essential, col_name, CompareFilter.CompareOp.EQUAL, flag_yes); filter.setFilterIfMissing(true); scan.setFilter(filter); scan.setLoadColumnFamiliesOnDemand(!slow); ResultScanner result_scanner = table.getScanner(scan); Result res; long rows_count = 0; while ((res = result_scanner.next()) != null) { rows_count++; } double timeSec = (System.nanoTime() - time) / 1000000000.0; result_scanner.close(); LOG.info((slow ? "Slow" : "Joined") + " scanner finished in " + Double.toString(timeSec) + " seconds, got " + Long.toString(rows_count/2) + " rows"); }
private ResultScanner buildScanner(String keyPrefix, String value, Table ht) throws IOException { // OurFilterList allFilters = new OurFilterList(); FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes .toBytes(value)); filter.setFilterIfMissing(true); allFilters.addFilter(filter); // allFilters.addFilter(new // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); Scan scan = new Scan(); scan.addFamily(Bytes.toBytes("trans-blob")); scan.addFamily(Bytes.toBytes("trans-type")); scan.addFamily(Bytes.toBytes("trans-date")); scan.addFamily(Bytes.toBytes("trans-tags")); scan.addFamily(Bytes.toBytes("trans-group")); scan.setFilter(allFilters); return ht.getScanner(scan); }
@Test public void testSingleColumnValueFilter() throws Exception { // null family/column SingleColumnValueFilter SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(null, null, CompareOperator.LESS, Bytes.toBytes("value")); assertTrue(singleColumnValueFilter.areSerializedFieldsEqual( ProtobufUtil.toFilter(ProtobufUtil.toFilter(singleColumnValueFilter)))); // non-null family/column SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(Bytes.toBytes("family"), Bytes.toBytes("qualifier"), CompareOperator.NOT_EQUAL, new NullComparator(), true, true); assertTrue(singleColumnValueFilter.areSerializedFieldsEqual( ProtobufUtil.toFilter(ProtobufUtil.toFilter(singleColumnValueFilter)))); }
@Test(expected = DoNotRetryIOException.class) public void testWithReverseScan() throws Exception { try (Connection con = TEST_UTIL.getConnection(); Admin admin = con.getAdmin()) { TableName tableName = TableName.valueOf(name.getMethodName()); TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf1")) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("cf2")) .build(); admin.createTable(tableDescriptor); try (Table table = con.getTable(tableName)) { SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("cf1"), Bytes.toBytes("col"), CompareOperator.EQUAL, Bytes.toBytes("val")); filter.setFilterIfMissing(true); // Reverse scan with loading CFs on demand Scan scan = new Scan(); scan.setFilter(filter); scan.setReversed(true); scan.setLoadColumnFamiliesOnDemand(true); try (ResultScanner scanner = table.getScanner(scan)) { // DoNotRetryIOException should occur scanner.next(); } } } } }
Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOperator.GREATER, new LongComparator(500)); scan.setFilter(filter);
protected Scan constructScan(byte[] valuePrefix) throws IOException { FilterList list = new FilterList(); Filter filter = new SingleColumnValueFilter(FAMILY_ZERO, COLUMN_ZERO, CompareOperator.EQUAL, new BinaryComparator(valuePrefix)); list.addFilter(filter); if (opts.filterAll) { list.addFilter(new FilterAllFilter()); } Scan scan = new Scan().setCaching(opts.caching).setCacheBlocks(opts.cacheBlocks) .setAsyncPrefetch(opts.asyncPrefetch).setReadType(opts.scanReadType) .setScanMetricsEnabled(true); if (opts.addColumns) { for (int column = 0; column < opts.columns; column++) { byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column); scan.addColumn(FAMILY_ZERO, qualifier); } } else { scan.addFamily(FAMILY_ZERO); } scan.setFilter(list); return scan; } }
/** * @param pbBytes A pb serialized {@link SingleColumnValueFilter} instance * @return An instance of {@link SingleColumnValueFilter} made from <code>bytes</code> * @throws org.apache.hadoop.hbase.exceptions.DeserializationException * @see #toByteArray */ public static SingleColumnValueFilter parseFrom(final byte [] pbBytes) throws DeserializationException { FilterProtos.SingleColumnValueFilter proto; try { proto = FilterProtos.SingleColumnValueFilter.parseFrom(pbBytes); } catch (InvalidProtocolBufferException e) { throw new DeserializationException(e); } final CompareOperator compareOp = CompareOperator.valueOf(proto.getCompareOp().name()); final org.apache.hadoop.hbase.filter.ByteArrayComparable comparator; try { comparator = ProtobufUtil.toComparator(proto.getComparator()); } catch (IOException ioe) { throw new DeserializationException(ioe); } return new SingleColumnValueFilter(proto.hasColumnFamily() ? proto.getColumnFamily() .toByteArray() : null, proto.hasColumnQualifier() ? proto.getColumnQualifier() .toByteArray() : null, compareOp, comparator, proto.getFilterIfMissing(), proto .getLatestVersionOnly()); }
@Test public void testLongComparator() throws IOException { Filter filter = new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOperator.GREATER, new LongComparator(100L)); KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(1L)); assertTrue("less than", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW); filter.reset(); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("less than", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(100L)); assertTrue("Equals 100", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW); filter.reset(); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("Equals 100", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(120L)); assertTrue("include 120", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); filter.reset(); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("include 120", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); }