@Override public ReturnCode filterCell(final Cell c) throws IOException { return this.filter.filterCell(c); }
@Override public ReturnCode filterCell(final Cell c) throws IOException { ReturnCode rc = filter.filterCell(c); changeFR(rc != ReturnCode.INCLUDE); return rc; }
@Override public ReturnCode filterCell(final Cell c) throws IOException { ReturnCode code = filter.filterCell(c); changeFAR(code != ReturnCode.INCLUDE); return code; }
/** * Attempt to filter out the keyvalue * @param c {@link Cell} on which to apply the filter * @return <tt>null</tt> if the key should not be written, otherwise returns the original * {@link Cell} */ public static Cell filterKv(Filter filter, Cell c) throws IOException { // apply the filter and skip this kv if the filter doesn't apply if (filter != null) { Filter.ReturnCode code = filter.filterCell(c); if (LOG.isTraceEnabled()) { LOG.trace("Filter returned:" + code + " for the cell:" + c); } // if its not an accept type, then skip this kv if (!(code.equals(Filter.ReturnCode.INCLUDE) || code .equals(Filter.ReturnCode.INCLUDE_AND_NEXT_COL))) { return null; } } return c; }
@Override public ReturnCode filterCell(Cell c) throws IOException { if (isEmpty()) { return ReturnCode.INCLUDE; } ReturnCode rc = ReturnCode.INCLUDE; this.seekHintFilters.clear(); for (int i = 0, n = filters.size(); i < n; i++) { Filter filter = filters.get(i); if (filter.filterAllRemaining()) { return ReturnCode.NEXT_ROW; } ReturnCode localRC; localRC = filter.filterCell(c); if (localRC == ReturnCode.SEEK_NEXT_USING_HINT) { seekHintFilters.add(filter); } rc = mergeReturnCode(rc, localRC); // Only when rc is INCLUDE* case, we should pass the cell to the following sub-filters. // otherwise we may mess up the global state (such as offset, count..) in the following // sub-filters. (HBASE-20565) if (!isIncludeRelatedReturnCode(rc)) { return rc; } } if (!seekHintFilters.isEmpty()) { return ReturnCode.SEEK_NEXT_USING_HINT; } return rc; }
localRC = filter.filterCell(c);
Bytes.toBytes("value")); Filter subFilter1 = Mockito.mock(FilterBase.class); Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL); Mockito.when(subFilter1.filterCell(kv2)).thenReturn(ReturnCode.NEXT_COL); Mockito.when(subFilter1.filterCell(kv3)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL); Mockito.when(subFilter1.filterCell(kv4)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL); Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SKIP); Mockito.when(subFilter2.filterCell(kv2)).thenReturn(ReturnCode.NEXT_ROW); Mockito.when(subFilter2.filterCell(kv3)).thenReturn(ReturnCode.NEXT_ROW); Mockito.when(subFilter2.filterCell(kv4)).thenReturn(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW); Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1)); Assert.assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv2)); Assert.assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv3)); Assert.assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv4)); Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.NEXT_ROW); Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT); filterList = new FilterList(Operator.MUST_PASS_ONE, subFilter1, subFilter2); Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1)); Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT); Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT); filterList = new FilterList(Operator.MUST_PASS_ONE, subFilter1, subFilter2); Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_1, VAL_1); assertTrue("includeAndSetFlag", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); assertTrue("flagIsSetSkipToNextRow", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW); System.out.println(filter.filterCell(cell)); assertTrue("includeFlagIsUnset", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); assertTrue("includeAndSetFlag", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_1, VAL_1); assertTrue("flagIsSetSkipToNextRow", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW);
filter.filterCell(cell));
private void substrFilterTests(Filter filter) throws Exception { KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("substrTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("substrTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_2); assertTrue("substrFalse", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("substrFalse", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("substrFilterAllRemaining", filter.filterAllRemaining()); assertFalse("substrFilterNotNull", filter.filterRow()); }
private void nullFilterTests(Filter filter) throws Exception { ((SingleColumnValueFilter) filter).setFilterIfMissing(true); KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("null1", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("null1", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("null1FilterRow", filter.filterRow()); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, Bytes.toBytes("qual2"), FULLSTRING_2); assertTrue("null2", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("null2", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertTrue("null2FilterRow", filter.filterRow()); }
/** * Test that the filter correctly drops rows without a corresponding timestamp * * @throws Exception */ @Test public void testFilterDropping() throws Exception { Filter filter = new DependentColumnFilter(FAMILIES[0], QUALIFIER); List<Cell> accepted = new ArrayList<>(); for(Cell val : testVals) { if(filter.filterCell(val) == ReturnCode.INCLUDE) { accepted.add(val); } } assertEquals("check all values accepted from filterCell", 5, accepted.size()); filter.filterRowCells(accepted); assertEquals("check filterRow(List<KeyValue>) dropped cell without corresponding column entry", 4, accepted.size()); // start do it again with dependent column dropping on filter = new DependentColumnFilter(FAMILIES[1], QUALIFIER, true); accepted.clear(); for(KeyValue val : testVals) { if(filter.filterCell(val) == ReturnCode.INCLUDE) { accepted.add(val); } } assertEquals("check the filtering column cells got dropped", 2, accepted.size()); filter.filterRowCells(accepted); assertEquals("check cell retention", 2, accepted.size()); }
private void regexFilterTests(Filter filter) throws Exception { KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("regexTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_2); assertTrue("regexFalse", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("regexFalse", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("regexFilterAllRemaining", filter.filterAllRemaining()); assertFalse("regexFilterNotNull", filter.filterRow()); }
private void regexPatternFilterTests(Filter filter) throws Exception { KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("regexTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("regexFilterAllRemaining", filter.filterAllRemaining()); assertFalse("regexFilterNotNull", filter.filterRow()); }
@Test public void testLongComparator() throws IOException { Filter filter = new SingleColumnValueFilter(COLUMN_FAMILY, COLUMN_QUALIFIER, CompareOperator.GREATER, new LongComparator(100L)); KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(1L)); assertTrue("less than", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW); filter.reset(); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("less than", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(100L)); assertTrue("Equals 100", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW); filter.reset(); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("Equals 100", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, Bytes.toBytes(120L)); assertTrue("include 120", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); filter.reset(); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("include 120", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); }
private void mpAllTest(Filter filterMPALL) throws Exception { /* Filter must do all below steps: * <ul> * <li>{@link #reset()}</li> * <li>{@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.</li> * <li>{@link #filterRowKey(byte[],int,int)} -> true to drop this row, * if false, we will also call</li> * <li>{@link #filterCell(org.apache.hadoop.hbase.KeyValue)} -> true to drop this cell</li> * <li>{@link #filterRow()} -> last chance to drop entire row based on the sequence of * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. * </li> * </ul> */ filterMPALL.reset(); assertFalse(filterMPALL.filterAllRemaining()); byte [] rowkey = Bytes.toBytes("yyyyyyyyy"); for (int i = 0; i < MAX_PAGES - 1; i++) { assertFalse(filterMPALL.filterRowKey(KeyValueUtil.createFirstOnRow(rowkey))); KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterCell(kv)); } filterMPALL.reset(); rowkey = Bytes.toBytes("z"); assertTrue(filterMPALL.filterRowKey(KeyValueUtil.createFirstOnRow(rowkey))); // Should fail here; row should be filtered out. KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey); assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterCell(kv)); }
assertTrue("otherColumn", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertTrue("testedMismatch", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW); assertTrue("otherColumn", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW);
KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterCell(kv)); assertFalse(filterMPONE.filterRow()); KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterCell(kv)); assertFalse(filterMPONE.filterRow()); KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterCell(kv)); assertFalse(filterMPONE.filterRow());
KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterCell(kv)); assertFalse(filterMPONE.filterRow()); KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), Bytes.toBytes(0)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPONE.filterCell(kv)); assertFalse(filterMPONE.filterRow()); kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(0), Bytes.toBytes(0)); assertFalse(Filter.ReturnCode.INCLUDE == filterMPONE.filterCell(kv)); assertFalse(filterMPONE.filterRow());
@Override public ReturnCode filterCell(final Cell c) throws IOException { ReturnCode code = filter.filterCell(c); changeFAR(code != ReturnCode.INCLUDE); return code; }