@Override public boolean filterRowKey(byte[] rowKey, int offset, int length) throws IOException { if (isEmpty()) { return super.filterRowKey(rowKey, offset, length); } boolean retVal = false; for (int i = 0, n = filters.size(); i < n; i++) { Filter filter = filters.get(i); if (filter.filterAllRemaining() || filter.filterRowKey(rowKey, offset, length)) { retVal = true; } } return retVal; }
filter.reset(); List<KeyValue> nkvs = new ArrayList<KeyValue>(tmp.size()); for (KeyValue kv : tmp) { if (filter.filterRowKey(kv.getBuffer(), kv.getRowOffset(), kv.getRowLength())) { filteredOnRowKey = true; break; Filter.ReturnCode filterResult = filter.filterKeyValue(kv); if (filterResult == Filter.ReturnCode.INCLUDE) { nkvs.add(kv); if (filter.hasFilterRow() && !filteredOnRowKey) { filter.filterRow(); if (filter.filterRow() || filteredOnRowKey) { nkvs.clear();
@Override public Cell getNextCellHint(Cell currentCell) throws IOException { if (isEmpty()) { return super.getNextCellHint(currentCell); } Cell maxHint = null; for (Filter filter : seekHintFilters) { if (filter.filterAllRemaining()) { continue; } Cell curKeyHint = filter.getNextCellHint(currentCell); if (maxHint == null) { maxHint = curKeyHint; continue; } if (this.compareCell(maxHint, curKeyHint) < 0) { maxHint = curKeyHint; } } return maxHint; }
public FilterRowRetCode filterRowCellsWithRet(List<Cell> kvs) throws IOException { //To fix HBASE-6429, //Filter with filterRow() returning true is incompatible with scan with limit //1. hasFilterRow() returns true, if either filterRow() or filterRow(kvs) is implemented. //2. filterRow() is merged with filterRow(kvs), //so that to make all those row related filtering stuff in the same function. this.filter.filterRowCells(kvs); if (!kvs.isEmpty()) { if (this.filter.filterRow()) { kvs.clear(); return FilterRowRetCode.EXCLUDE; } return FilterRowRetCode.INCLUDE; } return FilterRowRetCode.NOT_CALLED; }
private void mpAllTest(Filter filterMPALL) throws Exception { /* Filter must do all below steps: * <ul> * <li>{@link #reset()}</li> * <li>{@link #filterAllRemaining()} -> true indicates scan is over, false, keep going on.</li> * <li>{@link #filterRowKey(byte[],int,int)} -> true to drop this row, * if false, we will also call</li> * <li>{@link #filterCell(org.apache.hadoop.hbase.KeyValue)} -> true to drop this cell</li> * <li>{@link #filterRow()} -> last chance to drop entire row based on the sequence of * filterValue() calls. Eg: filter a row if it doesn't contain a specified column. * </li> * </ul> */ filterMPALL.reset(); assertFalse(filterMPALL.filterAllRemaining()); byte [] rowkey = Bytes.toBytes("yyyyyyyyy"); for (int i = 0; i < MAX_PAGES - 1; i++) { assertFalse(filterMPALL.filterRowKey(KeyValueUtil.createFirstOnRow(rowkey))); KeyValue kv = new KeyValue(rowkey, rowkey, Bytes.toBytes(i), Bytes.toBytes(i)); assertTrue(Filter.ReturnCode.INCLUDE == filterMPALL.filterCell(kv)); } filterMPALL.reset(); rowkey = Bytes.toBytes("z"); assertTrue(filterMPALL.filterRowKey(KeyValueUtil.createFirstOnRow(rowkey))); // Should fail here; row should be filtered out. KeyValue kv = new KeyValue(rowkey, rowkey, rowkey, rowkey); assertTrue(Filter.ReturnCode.NEXT_ROW == filterMPALL.filterCell(kv)); }
private void nullFilterTests(Filter filter) throws Exception { ((SingleColumnValueFilter) filter).setFilterIfMissing(true); KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("null1", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("null1", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("null1FilterRow", filter.filterRow()); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, Bytes.toBytes("qual2"), FULLSTRING_2); assertTrue("null2", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); buffer = cell.getBuffer(); c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("null2", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertTrue("null2FilterRow", filter.filterRow()); }
@Test public void testCompareFilter() throws Exception { Filter f = new RowFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("testRowOne-2"))); byte [] bytes = f.toByteArray(); Filter ff = RowFilter.parseFrom(bytes); assertNotNull(ff); }
Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL); Mockito.when(subFilter1.filterCell(kv2)).thenReturn(ReturnCode.NEXT_COL); Mockito.when(subFilter1.filterCell(kv3)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL); Mockito.when(subFilter1.filterCell(kv4)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL); Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SKIP); Mockito.when(subFilter2.filterCell(kv2)).thenReturn(ReturnCode.NEXT_ROW); Mockito.when(subFilter2.filterCell(kv3)).thenReturn(ReturnCode.NEXT_ROW); Mockito.when(subFilter2.filterCell(kv4)).thenReturn(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW); Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1)); Assert.assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv2)); Assert.assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv3)); Assert.assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv4)); Mockito.when(subFilter1.filterAllRemaining()).thenReturn(true); Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.NEXT_ROW); Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT); filterList = new FilterList(Operator.MUST_PASS_ONE, subFilter1, subFilter2); Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1)); Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT); Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT); filterList = new FilterList(Operator.MUST_PASS_ONE, subFilter1, subFilter2); Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
private void prefixRowTests(Filter filter, boolean lastFilterAllRemaining) throws Exception { for (char c = FIRST_CHAR; c <= LAST_CHAR; c++) { byte [] t = createRow(c); assertFalse("Failed with character " + c, filter.filterRowKey(KeyValueUtil.createFirstOnRow(t))); assertFalse(filter.filterAllRemaining()); } String yahooSite = "com.yahoo.www"; byte [] yahooSiteBytes = Bytes.toBytes(yahooSite); assertTrue("Failed with character " + yahooSite, filter.filterRowKey(KeyValueUtil.createFirstOnRow(yahooSiteBytes))); assertEquals(filter.filterAllRemaining(), lastFilterAllRemaining); }
private void writeResult(ImmutableBytesWritable key, Result result, Context context) throws IOException, InterruptedException { Put put = null; Delete delete = null; if (LOG.isTraceEnabled()) { LOG.trace("Considering the row." + Bytes.toString(key.get(), key.getOffset(), key.getLength())); } if (filter == null || !filter.filterRowKey(PrivateCellUtil.createFirstOnRow(key.get(), key.getOffset(), (short) key.getLength()))) { processKV(key, result, context, put, delete); } }
private void stopRowTests(Filter filter) throws Exception { assertFalse("Filtering on " + Bytes.toString(GOOD_ROW), filter.filterRowKey(KeyValueUtil.createFirstOnRow(GOOD_ROW))); assertFalse("Filtering on " + Bytes.toString(STOP_ROW), filter.filterRowKey(KeyValueUtil.createFirstOnRow(STOP_ROW))); assertTrue("Filtering on " + Bytes.toString(PAST_STOP_ROW), filter.filterRowKey(KeyValueUtil.createFirstOnRow(PAST_STOP_ROW))); assertTrue("FilterAllRemaining", filter.filterAllRemaining()); assertFalse("FilterNotNull", filter.filterRow()); }
private void regexPatternFilterTests(Filter filter) throws Exception { KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, FULLSTRING_1); assertTrue("regexTrue", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); byte[] buffer = cell.getBuffer(); Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length); assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); assertFalse("regexFilterAllRemaining", filter.filterAllRemaining()); assertFalse("regexFilterNotNull", filter.filterRow()); }
private void assertSeekAndHint(byte[] next, Filter f, byte[] rowHint, boolean filterAll) throws IOException { Cell c = new KeyValue(next, ByteUtil.EMPTY_BYTE_ARRAY, ByteUtil.EMPTY_BYTE_ARRAY, 0, ByteUtil.EMPTY_BYTE_ARRAY); assertTrue(f.filterKeyValue(c) == ReturnCode.SEEK_NEXT_USING_HINT); Cell h = f.getNextCellHint(c); byte[] hintBytes = rowHint; assertTrue(Bytes.equals(hintBytes, 0, hintBytes.length, h.getRowArray(), h.getRowOffset(), h.getRowLength())); assertEquals(filterAll, f.filterAllRemaining()); }
KeyValue c = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1)); kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_1)); kvs.add (new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1)); filter.filterRowCells(kvs); assertFalse("allRemainingWhenMatch", filter.filterAllRemaining()); filter.reset(); c = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); assertTrue("otherColumn", filter.filterCell(c) == Filter.ReturnCode.INCLUDE); c = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER, VAL_2); assertTrue("testedMismatch", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW); c = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); assertTrue("otherColumn", filter.filterCell(c) == Filter.ReturnCode.NEXT_ROW);
private void assertInclude(byte[] next, Filter f) throws IOException { Cell c = new KeyValue(next, ByteUtil.EMPTY_BYTE_ARRAY, ByteUtil.EMPTY_BYTE_ARRAY, 0, ByteUtil.EMPTY_BYTE_ARRAY); assertTrue(f.filterKeyValue(c) == ReturnCode.INCLUDE); assertFalse(f.filterAllRemaining()); }
cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_1, VAL_1); assertTrue("includeAndSetFlag", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); assertTrue("flagIsSetSkipToNextRow", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW); filter.reset(); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_3, VAL_1); System.out.println(filter.filterCell(cell)); assertTrue("includeFlagIsUnset", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_2, VAL_1); assertTrue("includeAndSetFlag", filter.filterCell(cell) == Filter.ReturnCode.INCLUDE); cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER_1, VAL_1); assertTrue("flagIsSetSkipToNextRow", filter.filterCell(cell) == Filter.ReturnCode.NEXT_ROW);
@Override public boolean hasFilterRow() { return this.filter.hasFilterRow(); }
@Override public boolean filterAllRemaining() throws IOException { return this.filterAllRemaining || this.filter.filterAllRemaining(); }
/** * A way to filter based on the column family, column qualifier and/or the column value. Return * code is described below. This allows filters to filter only certain number of columns, then * terminate without matching ever column. * * If filterRowKey returns true, filterCell needs to be consistent with it. * * filterCell can assume that filterRowKey has already been called for the row. * * If your filter returns <code>ReturnCode.NEXT_ROW</code>, it should return * <code>ReturnCode.NEXT_ROW</code> until {@link #reset()} is called just in case the caller calls * for the next row. * * Concrete implementers can signal a failure condition in their code by throwing an * {@link IOException}. * * @param c the Cell in question * @return code as described below * @throws IOException in case an I/O or an filter specific failure needs to be signaled. * @see Filter.ReturnCode */ public ReturnCode filterCell(final Cell c) throws IOException{ return filterKeyValue(c); }
private void testFiltersBeyondPageSize(final Filter f, final int pageSize) throws IOException { int count = 0; for (int i = 0; i < (pageSize * 2); i++) { boolean filterOut = f.filterRow(); if(filterOut) { break; } else { count++; } // If at last row, should tell us to skip all remaining if(count == pageSize) { assertTrue(f.filterAllRemaining()); } else { assertFalse(f.filterAllRemaining()); } } assertEquals(pageSize, count); }