public long estimatedSize() { return super.estimatedSize()+(8L*bloom2.length)+8L; }
/** * Returns 0 if this was a new addition, and 1 if this may have been and old value */ public int addValue(CharSequence value) { return add(value, bloom, bloomSeeds, bloomMask); }
public BloomFilter(long n, double p) { this(p, n, hashFunCount(n, p), bitsInFilter(n, p)); assert(p<=1); }
private void reportUnsupportedHeader() { StringBuilder headerName = new StringBuilder(); TrieParserReader.capturedFieldBytesAsUTF8(trieReader, 0, headerName); //in TRIE if we have any exact matches that run short must no pick anything. if (null==filter) { filter = new BloomFilter(10000, .00001); //32K } if (filter.mayContain(headerName)) { return;//do not report since we have already done so or we are overloaded with noise } else { filter.addValue(headerName); } StringBuilder headerValue = new StringBuilder(); TrieParserReader.capturedFieldBytesAsUTF8(trieReader, 1, headerValue); //in TRIE if we have any exact matches that run short must no pick anything. logger.info("WARNING unsupported header found: {}: {}",headerName, headerValue); ///logger.trace("length avail when parsed {}",len); }
private void logPrivateTopicTooShort(int token, Pipe<?> p) { String topic = publishPrivateTopics.getTopic(token); if (!topicsTooShort.mayContain(topic)) { logger.info("full pipe {}",p); logger.info("the private topic '{}' has become backed up, it may be too short. When it was defined it should be made to be longer.", topic); topicsTooShort.addValue(topic); } }
@Ignore public void testSize() { double x = .1; int c = 21; while (--c>=0) { BloomFilter filter = new BloomFilter(100L*1000L*1000L, x); long bytes = filter.estimatedSize(); System.out.println((bytes/(1024*1024))+"m "+x); x = x/10d; } }
@Override public void clear() { super.clear(); Arrays.fill(bloom, 0); }
private BloomFilter(double p, long n, int k, long m) { this(p,n,k,(int)Math.ceil(Math.log(m)/Math.log(2)),buildSeeds(k)); }
private static int hashFunCount(long n, double p) { return (int)Math.round(Math.log(2.0) * bitsInFilter(n,p) / n); }
@Test public void testAddAndFetch() { BloomFilter filter = new BloomFilter(1000, .00000001); //build up the filter with the known values. int i = testMessages1.length; while (--i>=0) { filter.addValue(testMessages1[i]); } //confirm expected assertTrue(filter.mayContain(testMessages1[0])); assertTrue(filter.mayContain(testMessages1[1])); assertTrue(filter.mayContain(testMessages1[2])); assertFalse(filter.mayContain(testMessages2[0])); assertFalse(filter.mayContain(testMessages2[1])); }
@Test public void testRollingTest() { String[] testValues = new String[] {"00","01","02","03","04","05","06","07","08","09","10"}; BloomFilter filter = new RollingBloomFilter(10, .00000001); int i = testValues.length; while (--i>=0) { filter.addValue(testValues[i]); assertTrue(filter.mayContain(testValues[i])); } // i = testValues.length; // while (--i>=0) { // System.out.println(" "+filter.mayContain(testValues[i])); // } assertTrue(filter.mayContain(testValues[0])); assertFalse(filter.mayContain(testValues[9])); }
public int addValue(byte[] source, int sourcePos, int sourceLen, int sourceMask) { return add(source, sourcePos, sourceLen, sourceMask, bloom, bloomSeeds, bloomMask); }