private static <T> boolean contains(ByteBuff bloomBuf, int bloomOffset, int bloomSize, Hash hash, int hashCount, HashKey<T> hashKey) { int hash1 = hash.hash(hashKey, 0); int bloomBitSize = bloomSize << 3; int hash2 = 0; int compositeHash = 0; if (randomGeneratorForTest == null) { // Production mode compositeHash = hash1; hash2 = hash.hash(hashKey, hash1); } for (int i = 0; i < hashCount; i++) { int hashLoc = (randomGeneratorForTest == null // Production mode ? Math.abs(compositeHash % bloomBitSize) // Test mode with "fake look-ups" to estimate "ideal false positive rate" : randomGeneratorForTest.nextInt(bloomBitSize)); compositeHash += hash2; if (!checkBit(hashLoc, bloomBuf, bloomOffset)) { return false; } } return true; }
/** * Get a singleton instance of hash function of a type * defined in the configuration. * @param conf current configuration * @return defined hash type, or null if type is invalid */ public static Hash getInstance(Configuration conf) { int type = getHashType(conf); return getInstance(type); }
/** * This utility method converts the name of the configured * hash type to a symbolic constant. * @param conf configuration * @return one of the predefined constants */ public static int getHashType(Configuration conf) { String name = conf.get("hbase.hash.type", "murmur"); return parseHashType(name); }
public BloomFilterChunk(int hashType, BloomType bloomType) { this.hashType = hashType; this.hash = Hash.getInstance(hashType); this.bloomType = bloomType; }
err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(), bloomType == BloomType.ROWCOL ? CellComparatorImpl.COMPARATOR : null, bloomType); writer.addInlineBlockWriter(bloomWriter);
/** * Loads bloom filter meta data from file input. * @param meta stored bloom meta data * @throws IllegalArgumentException meta data is invalid */ public BloomFilterChunk(DataInput meta) throws IOException, IllegalArgumentException { this.byteSize = meta.readInt(); this.hashCount = meta.readInt(); this.hashType = meta.readInt(); this.keyCount = meta.readInt(); this.maxKeys = this.keyCount; this.hash = Hash.getInstance(this.hashType); if (hash == null) { throw new IllegalArgumentException("Invalid hash type: " + hashType); } sanityCheck(); }
err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(), null, BloomType.ROW); writer.addInlineBlockWriter(bloomWriter);
public void add(Cell cell) { /* * For faster hashing, use combinatorial generation * http://www.eecs.harvard.edu/~kirsch/pubs/bbbf/esa06.pdf */ int hash1; int hash2; HashKey<Cell> hashKey; if (this.bloomType == BloomType.ROWCOL) { hashKey = new RowColBloomHashKey(cell); hash1 = this.hash.hash(hashKey, 0); hash2 = this.hash.hash(hashKey, hash1); } else { hashKey = new RowBloomHashKey(cell); hash1 = this.hash.hash(hashKey, 0); hash2 = this.hash.hash(hashKey, hash1); } setHashLoc(hash1, hash2); }
/** * Get a singleton instance of hash function of a type * defined in the configuration. * @param conf current configuration * @return defined hash type, or null if type is invalid */ public static Hash getInstance(Configuration conf) { int type = getHashType(conf); return getInstance(type); }
hash = Hash.getInstance(hashType); if (hash == null) { throw new IllegalArgumentException("Invalid hash type: " + hashType);
getBloomBlockSize(conf), err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(), bloomType == BloomType.ROWCOL ? KeyValue.KEY_COMPARATOR : Bytes.BYTES_RAWCOMPARATOR); } else if (maxKeys < tooBig) { BloomFilterWriter bloom = new ByteBloomFilter((int) maxKeys, err, Hash.getHashType(conf), maxFold); bloom.allocBloom(); return bloom;
/** * This utility method converts the name of the configured * hash type to a symbolic constant. * @param conf configuration * @return one of the predefined constants */ public static int getHashType(Configuration conf) { String name = conf.get("hbase.hash.type", "murmur"); return parseHashType(name); }
/** * @return the encodedName */ @InterfaceAudience.Private static String encodeRegionName(final byte [] regionName) { String encodedName; if (hasEncodedName(regionName)) { // region is in new format: // <tableName>,<startKey>,<regionIdTimeStamp>/encodedName/ encodedName = Bytes.toString(regionName, regionName.length - MD5_HEX_LENGTH - 1, MD5_HEX_LENGTH); } else { // old format region name. First hbase:meta region also // use this format.EncodedName is the JenkinsHash value. HashKey<byte[]> key = new ByteArrayHashKey(regionName, 0, regionName.length); int hashVal = Math.abs(JenkinsHash.getInstance().hash(key, 0)); encodedName = String.valueOf(hashVal); } return encodedName; }
/** * Get a singleton instance of hash function of a type * defined in the configuration. * @param conf current configuration * @return defined hash type, or null if type is invalid */ public static Hash getInstance(Configuration conf) { int type = getHashType(conf); return getInstance(type); }
/** Private constructor used by other constructors. */ private ByteBloomFilter(int hashType) { this.hashType = hashType; this.hash = Hash.getInstance(hashType); }
getBloomBlockSize(conf), err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(), Bytes.BYTES_RAWCOMPARATOR);
/** * This utility method converts the name of the configured * hash type to a symbolic constant. * @param conf configuration * @return one of the predefined constants */ public static int getHashType(Configuration conf) { String name = conf.get("hbase.hash.type", "murmur"); return parseHashType(name); }
LOG.info("Client=" + j + ", input=" + s); byte[] b = Bytes.toBytes(s); int hash = h.hash(new ByteArrayHashKey(b, 0, b.length), -1); m.put(hash, s);
/** * Get a singleton instance of hash function of a type * defined in the configuration. * @param conf current configuration * @return defined hash type, or null if type is invalid */ public static Hash getInstance(Configuration conf) { int type = getHashType(conf); return getInstance(type); }