public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments) { Preconditions.checkArgument(filterArguments.isEmpty(), "Expected 0 but got: %s", filterArguments.size()); return new FirstKeyOnlyFilter(); }
public static List<String> getTableEncodedRegionNamesForSerialReplication(Connection conn, TableName tableName) throws IOException { List<String> list = new ArrayList<>(); scanMeta(conn, getTableStartRowForMeta(tableName, QueryType.REPLICATION), getTableStopRowForMeta(tableName, QueryType.REPLICATION), QueryType.REPLICATION, new FirstKeyOnlyFilter(), Integer.MAX_VALUE, r -> { list.add(RegionInfo.encodeRegionName(r.getRow())); return true; }); return list; }
@Override public Map<String, Object> count(String task) { MappingConfig config = hbaseMapping.get(task); String hbaseTable = config.getHbaseMapping().getHbaseTable(); long rowCount = 0L; try { HTable table = (HTable) hbaseTemplate.getConnection().getTable(TableName.valueOf(hbaseTable)); Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); ResultScanner resultScanner = table.getScanner(scan); for (Result result : resultScanner) { rowCount += result.size(); } } catch (IOException e) { logger.error(e.getMessage(), e); } Map<String, Object> res = new LinkedHashMap<>(); res.put("hbaseTable", hbaseTable); res.put("count", rowCount); return res; }
/** * @param pbBytes A pb serialized {@link FirstKeyOnlyFilter} instance * @return An instance of {@link FirstKeyOnlyFilter} made from <code>bytes</code> * @throws org.apache.hadoop.hbase.exceptions.DeserializationException * @see #toByteArray */ public static FirstKeyOnlyFilter parseFrom(final byte [] pbBytes) throws DeserializationException { // There is nothing to deserialize. Why do this at all? try { FilterProtos.FirstKeyOnlyFilter.parseFrom(pbBytes); } catch (InvalidProtocolBufferException e) { throw new DeserializationException(e); } // Just return a new instance. return new FirstKeyOnlyFilter(); }
Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter());
/** * Sets filter {@link FilterBase} to the {@link Scan} instance. * If provided rowRangeList contains more than one element, * method sets filter which is instance of {@link MultiRowRangeFilter}. * Otherwise, method sets filter which is instance of {@link FirstKeyOnlyFilter}. * If rowRangeList contains exactly one element, startRow and stopRow are set to the scan. * @param scan * @param rowRangeList */ private static void setScanFilter(Scan scan, List<MultiRowRangeFilter.RowRange> rowRangeList) { final int size = rowRangeList == null ? 0 : rowRangeList.size(); if (size <= 1) { scan.setFilter(new FirstKeyOnlyFilter()); } if (size == 1) { MultiRowRangeFilter.RowRange range = rowRangeList.get(0); scan.setStartRow(range.getStartRow()); //inclusive scan.setStopRow(range.getStopRow()); //exclusive } else if (size > 1) { scan.setFilter(new MultiRowRangeFilter(rowRangeList)); } }
@Test public void testAddFilter() throws Exception { Filter filter1 = new FirstKeyOnlyFilter(); Filter filter2 = new FirstKeyOnlyFilter(); FilterList filterList = new FilterList(filter1, filter2); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Arrays.asList(filter1, filter2)); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2); filterList.addFilter(new FirstKeyOnlyFilter()); filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2)); filterList.addFilter(new FirstKeyOnlyFilter()); filterList.setReversed(false); FirstKeyOnlyFilter f = new FirstKeyOnlyFilter(); f.setReversed(true); try { filterList.addFilter(f); fail("The IllegalArgumentException should be thrown because the added filter is reversed"); } catch (IllegalArgumentException e) { } }
/** * Test partial Result re-assembly in the presence of different filters. The Results from the * partial scanner should match the Results returned from a scanner that receives all of the * results in one RPC to the server. The partial scanner is tested with a variety of different * result sizes (all of which are less than the size necessary to fetch an entire row) * @throws Exception */ @Test public void testPartialResultsWithColumnFilter() throws Exception { testPartialResultsWithColumnFilter(new FirstKeyOnlyFilter()); testPartialResultsWithColumnFilter(new ColumnPrefixFilter(Bytes.toBytes("testQualifier5"))); testPartialResultsWithColumnFilter(new ColumnRangeFilter(Bytes.toBytes("testQualifer1"), true, Bytes.toBytes("testQualifier7"), true)); Set<byte[]> qualifiers = new LinkedHashSet<>(); qualifiers.add(Bytes.toBytes("testQualifier5")); testPartialResultsWithColumnFilter(new FirstKeyValueMatchingQualifiersFilter(qualifiers)); }
/** * Tries to scan a row from passed region */ private void isSuccessfulScan(RegionInfo region) throws IOException { Scan scan = new Scan().withStartRow(region.getStartKey()).setRaw(true).setOneRowLimit() .setMaxResultSize(1L).setCaching(1).setFilter(new FirstKeyOnlyFilter()) .setCacheBlocks(false); try (Table table = conn.getTable(region.getTable()); ResultScanner scanner = table.getScanner(scan)) { scanner.next(); } catch (IOException e) { LOG.error("Could not scan region:" + region.getEncodedName(), e); throw e; } }
@Test public void testFirstKeyOnlyFilter() throws Exception { FirstKeyOnlyFilter firstKeyOnlyFilter = new FirstKeyOnlyFilter(); assertTrue(firstKeyOnlyFilter.areSerializedFieldsEqual( ProtobufUtil.toFilter(ProtobufUtil.toFilter(firstKeyOnlyFilter)))); }
scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan);
@Test public void testFilterAllRecords() throws IOException { Scan scan = new Scan(); scan.setBatch(1); scan.setCaching(1); // Filter out any records scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0]))); try (Table table = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) { try (ResultScanner s = table.getScanner(scan)) { assertNull(s.next()); } } }
@After public void tearDown() throws IOException { try (Table table = UTIL.getConnection().getTable(TableName.META_TABLE_NAME); ResultScanner scanner = table.getScanner(new Scan().addFamily(HConstants.CATALOG_FAMILY) .addFamily(HConstants.REPLICATION_BARRIER_FAMILY).setFilter(new FirstKeyOnlyFilter()))) { for (;;) { Result result = scanner.next(); if (result == null) { break; } TableName tableName = RegionInfo.getTable(result.getRow()); if (!tableName.isSystemTable()) { table.delete(new Delete(result.getRow())); } } } }
@Test public void testFirstKeyOnlyFilterAndBatch() throws IOException { Scan scan = new Scan(); scan.setFilter(new FirstKeyOnlyFilter()); scan.setBatch(1); InternalScanner scanner = REGION.getScanner(scan); List<Cell> results = new ArrayList<>(); for (int i = 0; i < NUM_ROWS; i++) { results.clear(); scanner.next(results); assertEquals(1, results.size()); Cell cell = results.get(0); assertArrayEquals(ROWS[i], Bytes.copy(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength())); } assertFalse(scanner.next(results)); scanner.close(); }
@Test public void testConstruction() { FirstKeyOnlyFilter f1 = new FirstKeyOnlyFilter(); FirstKeyOnlyFilter f2 = new FirstKeyOnlyFilter(); f1.setReversed(true); f2.setReversed(false); try { FilterList ff = new FilterList(f1, f2); fail("The IllegalArgumentException should be thrown"); } catch (IllegalArgumentException e) { } try { FilterList ff = new FilterList(Arrays.asList(f1, f2)); fail("The IllegalArgumentException should be thrown because the added filter is reversed"); } catch (IllegalArgumentException e) { } try { FilterList ff = new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.asList(f1, f2)); fail("The IllegalArgumentException should be thrown because the added filter is reversed"); } catch (IllegalArgumentException e) { } try { FilterList ff = new FilterList(FilterList.Operator.MUST_PASS_ALL, f1, f2); fail("The IllegalArgumentException should be thrown because the added filter is reversed"); } catch (IllegalArgumentException e) { } } /**
@Test public void testMayHaveMoreCellsInRowReturnsTrueAndSetBatch() throws IOException { Table table = createTestTable(TableName.valueOf(name.getMethodName()), ROWS, FAMILIES, QUALIFIERS, VALUE); Scan scan = new Scan(); scan.setBatch(1); scan.setFilter(new FirstKeyOnlyFilter()); ResultScanner scanner = table.getScanner(scan); Result result; while ((result = scanner.next()) != null) { assertTrue(result.rawCells() != null); assertEquals(1, result.rawCells().length); } }
@Test public void testFirstKeyOnlyFilter() throws IOException { Scan s = new Scan(); s.setFilter(new FirstKeyOnlyFilter()); // Expected KVs, the first KV from each of the remaining 6 rows KeyValue [] kvs = { new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) }; verifyScanFull(s, kvs); }
@Test public void testFirstKeyOnlyFilter() throws Exception { Scan s = new Scan(); s.setFilter(new FirstKeyOnlyFilter()); // Expected KVs, the first KV from each of the remaining 6 rows KeyValue [] kvs = { new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]), new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]), new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]) }; verifyScanFull(s, kvs); } }
@Test public void testReverseScanWithPadding() throws Exception { byte[] terminator = new byte[] { -1 }; byte[] row1 = Bytes.add(invert(Bytes.toBytes("a")), terminator); byte[] row2 = Bytes.add(invert(Bytes.toBytes("ab")), terminator); byte[] row3 = Bytes.add(invert(Bytes.toBytes("b")), terminator); Put put1 = new Put(row1); put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put2 = new Put(row2); put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); Put put3 = new Put(row3); put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY); region.put(put1); region.put(put2); region.put(put3); region.flush(true); Scan scan = new Scan(); scan.setCacheBlocks(false); scan.setReversed(true); scan.setFilter(new FirstKeyOnlyFilter()); scan.addFamily(cfName); RegionScanner scanner = region.getScanner(scan); List<Cell> res = new ArrayList<>(); int count = 1; while (scanner.next(res)) { count++; } assertEquals(3, count); }
filter = new FirstKeyOnlyFilter(); testRowsFilteredMetric(baseScan, filter, 0);