private NavigableMap<byte[], NavigableMap<byte[], Update>> applyFilter( NavigableMap<byte[], NavigableMap<byte[], Update>> bufferMap, @Nullable Filter filter) { if (filter == null) { return bufferMap; } // todo: currently we support only FuzzyRowFilter as an experimental feature if (filter instanceof FuzzyRowFilter) { NavigableMap<byte[], NavigableMap<byte[], Update>> result = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], NavigableMap<byte[], Update>> entry : bufferMap.entrySet()) { if (FuzzyRowFilter.ReturnCode.INCLUDE == ((FuzzyRowFilter) filter).filterRow(entry.getKey())) { result.put(entry.getKey(), entry.getValue()); } } return result; } else { throw new DataSetException("Unknown filter type: " + filter); } }
private void setFilterIfNeeded(ScanBuilder scan, @Nullable Filter filter) { if (filter == null) { return; } if (filter instanceof FuzzyRowFilter) { FuzzyRowFilter fuzzyRowFilter = (FuzzyRowFilter) filter; List<Pair<byte[], byte[]>> fuzzyPairs = Lists.newArrayListWithExpectedSize(fuzzyRowFilter.getFuzzyKeysData().size()); for (ImmutablePair<byte[], byte[]> pair : fuzzyRowFilter.getFuzzyKeysData()) { fuzzyPairs.add(Pair.newPair(pair.getFirst(), pair.getSecond())); } scan.setFilter(new org.apache.hadoop.hbase.filter.FuzzyRowFilter(fuzzyPairs)); } else { throw new IllegalArgumentException("Unsupported filter: " + filter); } }
if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) { result[i] = row[offset + i]; if (!isMax(row[i])) {
/** * Returns mapping of all that match the given keySet provided they pass the combinedFilter predicate * for default COLUMN * * @param keySet row key set * @param typeOfT the type of the result * @param limit limit number of result * @param combinedFilter filter for key * @return map of row key to result */ public <T> Map<MDSKey, T> listKV(Set<MDSKey> keySet, Type typeOfT, int limit, @Nullable Predicate<KeyValue<T>> combinedFilter) { // Sort fuzzy keys List<MDSKey> sortedKeys = Lists.newArrayList(keySet); Collections.sort(sortedKeys); // Scan using fuzzy filter byte[] startKey = sortedKeys.get(0).getKey(); byte[] stopKey = Bytes.stopKeyForPrefix(sortedKeys.get(sortedKeys.size() - 1).getKey()); List<ImmutablePair<byte [], byte []>> fuzzyKeys = new ArrayList<>(); for (MDSKey key : sortedKeys) { fuzzyKeys.add(getFuzzyKeyFor(key)); } Scan scan = new Scan(startKey, stopKey, new FuzzyRowFilter(fuzzyKeys)); return listCombinedFilterKV(scan, typeOfT, limit, combinedFilter); }
byte[] mask = codec.createFuzzyRowMask(dimensionValues, "myMetric"); rowKey = codec.createRowKey(dimensionValues, "myMetric", ts); FuzzyRowFilter filter = new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(rowKey, mask))); new DimensionValue("dimension3", "value3")); rowKey = codec.createRowKey(dimensionValues, "myMetric", ts); Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey)); new DimensionValue("dimension3", "value3")); rowKey = codec.createRowKey(dimensionValues, "myMetric", ts); Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey)); new DimensionValue("dimension3", "value13")); rowKey = codec.createRowKey(dimensionValues, "myMetric", ts); Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey)); Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey)); new DimensionValue("dimension3", "value3")); rowKey = codec.createRowKey(dimensionValues, "myMetric", ts); Assert.assertEquals(FuzzyRowFilter.ReturnCode.INCLUDE, filter.filterRow(rowKey)); new DimensionValue("dimension3", "value3")); rowKey = codec.createRowKey(dimensionValues, "myMetric2", ts); Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey)); Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey)); Assert.assertTrue(FuzzyRowFilter.ReturnCode.INCLUDE != filter.filterRow(rowKey));
FuzzyRowFilter.ReturnCode code = filter.filterRow(result.getFirst()); switch (code) { case DONE: { byte[] seekToRow = filter.getNextRowHint(result.getFirst()); iterator.seek(createStartKey(seekToRow)); continue;
public byte[] getNextRowHint(byte[] rowKey) { byte[] nextRowKey = null; // Searching for the "smallest" row key that satisfies at least one fuzzy row key for (ImmutablePair<byte[], byte[]> fuzzyData : fuzzyKeysData) { byte[] nextRowKeyCandidate = getNextForFuzzyRule(rowKey, fuzzyData.getFirst(), fuzzyData.getSecond()); if (nextRowKeyCandidate == null) { continue; } if (nextRowKey == null || Bytes.compareTo(nextRowKeyCandidate, nextRowKey) < 0) { nextRowKey = nextRowKeyCandidate; } } if (nextRowKey == null) { // SHOULD NEVER happen // TODO: is there a better way than throw exception? (stop the scanner?) throw new IllegalStateException("No next row key that satisfies fuzzy exists when" + " getNextKeyHint() is invoked." + " Filter: " + this.toString() + " RowKey: " + Bytes.toStringBinary(rowKey)); } return nextRowKey; }
private static SatisfiesCode satisfies(byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { return satisfies(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta); }
if (fuzzyKeyMeta[i] == 1 && !isMax(fuzzyKeyBytes[i])) { nextRowKeyCandidateExists = true;
/** * Returns mapping of all that match the given keySet provided they pass the combinedFilter predicate * for default COLUMN * * @param keySet row key set * @param typeOfT the type of the result * @param limit limit number of result * @param combinedFilter filter for key * @return map of row key to result */ public <T> Map<MDSKey, T> listKV(Set<MDSKey> keySet, Type typeOfT, int limit, @Nullable Predicate<KeyValue<T>> combinedFilter) { // Sort fuzzy keys List<MDSKey> sortedKeys = Lists.newArrayList(keySet); Collections.sort(sortedKeys); // Scan using fuzzy filter byte[] startKey = sortedKeys.get(0).getKey(); byte[] stopKey = Bytes.stopKeyForPrefix(sortedKeys.get(sortedKeys.size() - 1).getKey()); List<ImmutablePair<byte [], byte []>> fuzzyKeys = new ArrayList<>(); for (MDSKey key : sortedKeys) { fuzzyKeys.add(getFuzzyKeyFor(key)); } Scan scan = new Scan(startKey, stopKey, new FuzzyRowFilter(fuzzyKeys)); return listCombinedFilterKV(scan, typeOfT, limit, combinedFilter); }
FuzzyRowFilter.ReturnCode code = filter.filterRow(result.getFirst()); switch (code) { case DONE: { byte[] seekToRow = filter.getNextRowHint(result.getFirst()); iterator.seek(createStartKey(seekToRow)); continue;
public byte[] getNextRowHint(byte[] rowKey) { byte[] nextRowKey = null; // Searching for the "smallest" row key that satisfies at least one fuzzy row key for (ImmutablePair<byte[], byte[]> fuzzyData : fuzzyKeysData) { byte[] nextRowKeyCandidate = getNextForFuzzyRule(rowKey, fuzzyData.getFirst(), fuzzyData.getSecond()); if (nextRowKeyCandidate == null) { continue; } if (nextRowKey == null || Bytes.compareTo(nextRowKeyCandidate, nextRowKey) < 0) { nextRowKey = nextRowKeyCandidate; } } if (nextRowKey == null) { // SHOULD NEVER happen // TODO: is there a better way than throw exception? (stop the scanner?) throw new IllegalStateException("No next row key that satisfies fuzzy exists when" + " getNextKeyHint() is invoked." + " Filter: " + this.toString() + " RowKey: " + Bytes.toStringBinary(rowKey)); } return nextRowKey; }
private static SatisfiesCode satisfies(byte[] row, byte[] fuzzyKeyBytes, byte[] fuzzyKeyMeta) { return satisfies(row, 0, row.length, fuzzyKeyBytes, fuzzyKeyMeta); }
if (fuzzyKeyMeta[i] == 1 && !isMax(fuzzyKeyBytes[i])) { nextRowKeyCandidateExists = true;
private FuzzyRowFilter createFuzzyRowFilter(FactScan scan, byte[] startRow) { // we need to always use a fuzzy row filter as it is the only one to do the matching of values // if we are querying only one measure, we will use fixed measureName for filter, // if there are no measures or more than one measures to query we use `ANY` fuzzy filter. String measureName = (scan.getMeasureNames().size() == 1) ? scan.getMeasureNames().iterator().next() : null; byte[] fuzzyRowMask = codec.createFuzzyRowMask(scan.getDimensionValues(), measureName); // note: we can use startRow, as it will contain all "fixed" parts of the key needed return new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(startRow, fuzzyRowMask))); }
private NavigableMap<byte[], NavigableMap<byte[], byte[]>> applyFilter( NavigableMap<byte[], NavigableMap<byte[], byte[]>> map, @Nullable Filter filter) { if (filter == null) { return map; } // todo: currently we support only FuzzyRowFilter as an experimental feature if (filter instanceof FuzzyRowFilter) { NavigableMap<byte[], NavigableMap<byte[], byte[]>> result = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> entry : map.entrySet()) { if (FuzzyRowFilter.ReturnCode.INCLUDE == ((FuzzyRowFilter) filter).filterRow(entry.getKey())) { result.put(entry.getKey(), entry.getValue()); } } return result; } else { throw new DataSetException("Unknown filter type: " + filter); } }
if (i >= fuzzyKeyMeta.length || fuzzyKeyMeta[i] == 1) { result[i] = row[offset + i]; if (!isMax(row[i])) {
public ReturnCode filterRow(byte[] rowKey) { // assigning "worst" result first and looking for better options SatisfiesCode bestOption = SatisfiesCode.NO_NEXT; for (ImmutablePair<byte[], byte[]> fuzzyData : fuzzyKeysData) { SatisfiesCode satisfiesCode = satisfies(rowKey, fuzzyData.getFirst(), fuzzyData.getSecond()); if (satisfiesCode == SatisfiesCode.YES) { return ReturnCode.INCLUDE; } if (satisfiesCode == SatisfiesCode.NEXT_EXISTS) { bestOption = SatisfiesCode.NEXT_EXISTS; } } if (bestOption == SatisfiesCode.NEXT_EXISTS) { return ReturnCode.SEEK_NEXT_USING_HINT; } // the only unhandled SatisfiesCode is NO_NEXT, i.e. we are done return ReturnCode.DONE; }
private void setFilterIfNeeded(ScanBuilder scan, @Nullable Filter filter) { if (filter == null) { return; } if (filter instanceof FuzzyRowFilter) { FuzzyRowFilter fuzzyRowFilter = (FuzzyRowFilter) filter; List<Pair<byte[], byte[]>> fuzzyPairs = Lists.newArrayListWithExpectedSize(fuzzyRowFilter.getFuzzyKeysData().size()); for (ImmutablePair<byte[], byte[]> pair : fuzzyRowFilter.getFuzzyKeysData()) { fuzzyPairs.add(Pair.newPair(pair.getFirst(), pair.getSecond())); } scan.setFilter(new org.apache.hadoop.hbase.filter.FuzzyRowFilter(fuzzyPairs)); } else { throw new IllegalArgumentException("Unsupported filter: " + filter); } }
private FuzzyRowFilter createFuzzyRowFilter(FactScan scan, byte[] startRow) { // we need to always use a fuzzy row filter as it is the only one to do the matching of values // if we are querying only one measure, we will use fixed measureName for filter, // if there are no measures or more than one measures to query we use `ANY` fuzzy filter. String measureName = (scan.getMeasureNames().size() == 1) ? scan.getMeasureNames().iterator().next() : null; byte[] fuzzyRowMask = codec.createFuzzyRowMask(scan.getDimensionValues(), measureName); // note: we can use startRow, as it will contain all "fixed" parts of the key needed return new FuzzyRowFilter(ImmutableList.of(new ImmutablePair<>(startRow, fuzzyRowMask))); }