Refine search
public void testPutReturnValue() { for (int i = 0; i < 10; i++) { BloomFilter<String> bf = BloomFilter.create(Funnels.unencodedCharsFunnel(), 100); for (int j = 0; j < 10; j++) { String value = new Object().toString(); boolean mightContain = bf.mightContain(value); boolean put = bf.put(value); assertTrue(mightContain != put); } } }
@VisibleForTesting static <T> BloomFilter<T> create( Funnel<? super T> funnel, long expectedInsertions, double fpp, Strategy strategy) { checkNotNull(funnel); checkArgument( expectedInsertions >= 0, "Expected insertions (%s) must be >= 0", expectedInsertions); checkArgument(fpp > 0.0, "False positive probability (%s) must be > 0.0", fpp); checkArgument(fpp < 1.0, "False positive probability (%s) must be < 1.0", fpp); checkNotNull(strategy); if (expectedInsertions == 0) { expectedInsertions = 1; } /* * TODO(user): Put a warning in the javadoc about tiny fpp values, since the resulting size * is proportional to -log(p), but there is not much of a point after all, e.g. * optimalM(1000, 0.0000000000000001) = 76680 which is less than 10kb. Who cares! */ long numBits = optimalNumOfBits(expectedInsertions, fpp); int numHashFunctions = optimalNumOfHashFunctions(expectedInsertions, numBits); try { return new BloomFilter<T>(new LockFreeBitArray(numBits), numHashFunctions, funnel, strategy); } catch (IllegalArgumentException e) { throw new IllegalArgumentException("Could not create BloomFilter of " + numBits + " bits", e); } }
@AndroidIncompatible // slow public void testBitSize() { double fpp = 0.03; for (int i = 1; i < 10000; i++) { long numBits = BloomFilter.optimalNumOfBits(i, fpp); int arraySize = Ints.checkedCast(LongMath.divide(numBits, 64, RoundingMode.CEILING)); assertEquals( arraySize * Long.SIZE, BloomFilter.create(Funnels.unencodedCharsFunnel(), i, fpp).bitSize()); } }
@Override public boolean isDuplicate(Request request, Task task) { boolean isDuplicate = bloomFilter.mightContain(getUrl(request)); if (!isDuplicate) { bloomFilter.put(getUrl(request)); counter.incrementAndGet(); } return isDuplicate; }
public void testApproximateElementCount() { int numInsertions = 1000; BloomFilter<Integer> bf = BloomFilter.create(Funnels.integerFunnel(), numInsertions); bf.put(-1); for (int i = 0; i < numInsertions; i++) { bf.put(i); } assertApproximateElementCountGuess(bf, numInsertions); }
public void testJavaSerialization() { BloomFilter<byte[]> bf = BloomFilter.create(Funnels.byteArrayFunnel(), 100); for (int i = 0; i < 10; i++) { bf.put(Ints.toByteArray(i)); } BloomFilter<byte[]> copy = SerializableTester.reserialize(bf); for (int i = 0; i < 10; i++) { assertTrue(copy.mightContain(Ints.toByteArray(i))); } assertEquals(bf.expectedFpp(), copy.expectedFpp()); SerializableTester.reserializeAndAssert(bf); }
public void testCreateAndCheckBloomFilterWithKnownUtf8FalsePositives64() { int numInsertions = 1000000; BloomFilter<String> bf = BloomFilter.create( Funnels.stringFunnel(UTF_8), numInsertions, 0.03, bf.put(Integer.toString(i)); assertTrue(bf.mightContain(Integer.toString(i))); for (int i = 1; i < 900; i += 2) { if (!falsePositives.contains(i)) { assertFalse("BF should not contain " + i, bf.mightContain(Integer.toString(i))); int numFpp = 0; for (int i = 1; i < numInsertions * 2; i += 2) { if (bf.mightContain(Integer.toString(i))) { numFpp++; double actualReportedFpp = bf.expectedFpp(); assertEquals(expectedReportedFpp, actualReportedFpp, 0.00033);
public void testPutAll() { int element1 = 1; int element2 = 2; BloomFilter<Integer> bf1 = BloomFilter.create(Funnels.integerFunnel(), 100); bf1.put(element1); assertTrue(bf1.mightContain(element1)); assertFalse(bf1.mightContain(element2)); BloomFilter<Integer> bf2 = BloomFilter.create(Funnels.integerFunnel(), 100); bf2.put(element2); assertFalse(bf2.mightContain(element1)); assertTrue(bf2.mightContain(element2)); assertTrue(bf1.isCompatible(bf2)); bf1.putAll(bf2); assertTrue(bf1.mightContain(element1)); assertTrue(bf1.mightContain(element2)); assertFalse(bf2.mightContain(element1)); assertTrue(bf2.mightContain(element2)); }
/** * @deprecated Provided only to satisfy the {@link Predicate} interface; use {@link #mightContain} * instead. */ @Deprecated @Override public boolean apply(T input) { return mightContain(input); }
public void testCustomSerialization() throws Exception { Funnel<byte[]> funnel = Funnels.byteArrayFunnel(); BloomFilter<byte[]> bf = BloomFilter.create(funnel, 100); for (int i = 0; i < 100; i++) { bf.put(Ints.toByteArray(i)); } ByteArrayOutputStream out = new ByteArrayOutputStream(); bf.writeTo(out); assertEquals(bf, BloomFilter.readFrom(new ByteArrayInputStream(out.toByteArray()), funnel)); }
public void testCollector() { BloomFilter<String> bf1 = BloomFilter.create(Funnels.unencodedCharsFunnel(), 100); bf1.put("1"); bf1.put("2"); assertEquals( bf1, Stream.of("1", "2") .collect(BloomFilter.toBloomFilter(Funnels.unencodedCharsFunnel(), 100))); assertEquals( bf1, Stream.of("2", "1") .collect(BloomFilter.toBloomFilter(Funnels.unencodedCharsFunnel(), 100))); }
public void testExpectedFpp() { BloomFilter<Object> bf = BloomFilter.create(HashTestUtils.BAD_FUNNEL, 10, 0.03); double fpp = bf.expectedFpp(); assertEquals(0.0, fpp); // usually completed in less than 200 iterations while (fpp != 1.0) { boolean changed = bf.put(new Object()); double newFpp = bf.expectedFpp(); // if changed, the new fpp is strictly higher, otherwise it is the same assertTrue(changed ? newFpp > fpp : newFpp == fpp); fpp = newFpp; } }
@AndroidIncompatible // OutOfMemoryError public void testLargeNumberOfInsertions() { // We use horrible FPPs here to keep Java from OOM'ing BloomFilter<String> unused = BloomFilter.create(Funnels.unencodedCharsFunnel(), Integer.MAX_VALUE / 2, 0.29); unused = BloomFilter.create(Funnels.unencodedCharsFunnel(), 45L * Integer.MAX_VALUE, 0.99); }
protected BloomFilter<CharSequence> rebuildBloomFilter() { counter = new AtomicInteger(0); return BloomFilter.create(Funnels.stringFunnel(Charset.defaultCharset()), expectedInsertions, fpp); }
public void testEquals() { BloomFilter<String> bf1 = BloomFilter.create(Funnels.unencodedCharsFunnel(), 100); bf1.put("1"); bf1.put("2"); BloomFilter<String> bf2 = BloomFilter.create(Funnels.unencodedCharsFunnel(), 100); bf2.put("1"); bf2.put("2"); new EqualsTester().addEqualityGroup(bf1, bf2).testEquals(); bf2.put("3"); new EqualsTester().addEqualityGroup(bf1).addEqualityGroup(bf2).testEquals(); }
private static void checkSanity(BloomFilter<Object> bf) { assertFalse(bf.mightContain(new Object())); assertFalse(bf.apply(new Object())); for (int i = 0; i < 100; i++) { Object o = new Object(); bf.put(o); assertTrue(bf.mightContain(o)); assertTrue(bf.apply(o)); } }
public void testPutAllWithSelf() { BloomFilter<Integer> bf1 = BloomFilter.create(Funnels.integerFunnel(), 1); try { assertFalse(bf1.isCompatible(bf1)); bf1.putAll(bf1); fail(); } catch (IllegalArgumentException expected) { } }
/** Tests that we always get a non-negative optimal size. */ public void testOptimalSize() { for (int n = 1; n < 1000; n++) { for (double fpp = Double.MIN_VALUE; fpp < 1.0; fpp += 0.001) { assertTrue(BloomFilter.optimalNumOfBits(n, fpp) >= 0); } } // some random values Random random = new Random(0); for (int repeats = 0; repeats < 10000; repeats++) { assertTrue(BloomFilter.optimalNumOfBits(random.nextInt(1 << 16), random.nextDouble()) >= 0); } // and some crazy values (this used to be capped to Integer.MAX_VALUE, now it can go bigger assertEquals(3327428144502L, BloomFilter.optimalNumOfBits(Integer.MAX_VALUE, Double.MIN_VALUE)); try { BloomFilter<String> unused = BloomFilter.create(HashTestUtils.BAD_FUNNEL, Integer.MAX_VALUE, Double.MIN_VALUE); fail("we can't represent such a large BF!"); } catch (IllegalArgumentException expected) { assertThat(expected) .hasMessageThat() .isEqualTo("Could not create BloomFilter of 3327428144502 bits"); } }
/** Sanity checking with many combinations of false positive rates and expected insertions */ public void testBasic() { for (double fpr = 0.0000001; fpr < 0.1; fpr *= 10) { for (int expectedInsertions = 1; expectedInsertions <= 10000; expectedInsertions *= 10) { checkSanity(BloomFilter.create(HashTestUtils.BAD_FUNNEL, expectedInsertions, fpr)); } } }
@Override public boolean put(long e) { return bloomFilter.put(e); }