public void recordSkew( InetSocketAddress originator_address, long skew ) { byte[] bytes = AddressUtils.getAddressBytes( originator_address ); if ( skew_originator_bloom.contains( bytes)){ //System.out.println( "skipping skew: " + originator_address ); return; } skew_originator_bloom.add( bytes ); //System.out.println( "adding skew: " + originator_address + "/" + skew ); int i_skew = skew<Integer.MAX_VALUE?(int)skew:(Integer.MAX_VALUE-1); // no sync here as not important so ensure things work ok int pos = skew_pos; skew_values[ pos++ ] = i_skew; if ( pos == SKEW_VALUE_MAX ){ pos = 0; } skew_pos = pos; }
protected void serialiseToMap( Map<String,Object> x ) { synchronized( filters ){ String cla = this.getClass().getName(); if ( cla.startsWith( BloomFilterImpl.MY_PACKAGE )){ cla = cla.substring( BloomFilterImpl.MY_PACKAGE.length()); } x.put( "_impl", cla ); List<Map<String,Object>> list = new ArrayList<>(); for ( BloomFilter filter: filters ){ list.add( filter.serialiseToMap()); } x.put( "list", list ); x.put( "index", new Long( current_filter_index )); } }
@Override public void contactKnown( byte[] node_id, DHTRouterContactAttachment attachment, boolean force ) { // especially for small DHTs we don't want to prevent a contact from being re-added as long as they've been away for // a bit if ( SystemTime.getMonotonousTime() - recent_contact_bloom.getStartTimeMono() > 10*60*1000 ){ recent_contact_bloom.clear(); } if ( recent_contact_bloom.contains( node_id )){ if ( !force ){ return; } } recent_contact_bloom.add( node_id ); addContact( node_id, attachment, false ); }
int filter_size = current_filter.getSize(); int filter_entries = current_filter.getEntryCount(); int r = filters[i%filters.length].add( value ); if ( current_filter.getEntryCount() > limit ){ filters[current_filter_index] = current_filter.getReplica();
@Override public void deactivateRequest( InetSocketAddress address ) { BloomFilter bloom = activation_bloom; if ( bloom != null ){ byte[] address_bytes = AddressUtils.getAddressBytes( address ); int count = bloom.count( address_bytes); for (int i=0;i<count;i++){ bloom.remove( address_bytes ); } activation_count = bloom.getEntryCount(); } }
protected void addPersistentlyDeleted( RelatedContent[] content ) { if ( content.length == 0 ){ return; } List<byte[]> entries = loadPersistentlyDeleted(); List<byte[]> new_keys = new ArrayList<>(content.length); for ( RelatedContent rc: content ){ byte[] key = getPermDelKey( rc ); new_keys.add( key ); entries.add( key ); } Map<String,Object> map = new HashMap<>(); map.put( "entries", entries ); FileUtil.writeResilientConfigFile( PERSIST_DEL_FILE, map ); if ( persist_del_bloom != null ){ if ( persist_del_bloom.getSize() / ( persist_del_bloom.getEntryCount() + content.length ) < 10 ){ persist_del_bloom = BloomFilterFactory.createAddOnly( Math.max( PD_BLOOM_INITIAL_SIZE, persist_del_bloom.getSize() *10 + PD_BLOOM_INCREMENT_SIZE + content.length )); for ( byte[] k: entries ){ persist_del_bloom.add( k ); } }else{ for ( byte[] k: new_keys ){ persist_del_bloom.add( k ); } } } }
boolean dup = harvest_op_requester_bloom.contains( originator_bytes ); logSearch( "Received remote request: " + BDecoder.decodeStrings( request ) + " from " + originator.getAddress() + "/" + originator.getNetwork() + ", dup=" + dup + ", bs=" + harvest_op_requester_bloom.getEntryCount()); harvest_op_requester_bloom.add( originator_bytes ); response.put( "f", filter.serialiseToMap()); if ( existing_size != filter.getEntryCount()){ response.put( "f", filter.serialiseToMap()); int hits = harvest_se_requester_bloom.count( originator_bytes ); logSearch( "Received remote search: '" + term + "' from " + originator.getAddress() + ", hits=" + hits + ", bs=" + harvest_se_requester_bloom.getEntryCount()); harvest_se_requester_bloom.add( originator_bytes );
protected void findContacts() { DHTTransportContact[] reachables = dht.getTransport().getReachableContacts(); for (int i=0;i<reachables.length;i++){ DHTTransportContact contact = reachables[i]; byte[] address = contact.getAddress().getAddress().getAddress(); if ( tried_bloom == null || tried_bloom.getEntryCount() > 500 ){ tried_bloom = BloomFilterFactory.createAddOnly( 4096 ); } if ( !tried_bloom.contains( address )){ tried_bloom.add( address ); synchronized( pending_contacts ){ potentialPing ping = new potentialPing( contact, DHTNetworkPositionManager.estimateRTT( contact.getNetworkPositions(), dht.getTransport().getLocalContact().getNetworkPositions())); pending_contacts.add( 0, ping ); if ( pending_contacts.size() > 60 ){ pending_contacts.removeLast(); } } } } }
@Override public void sentTo( DHTTransportContact contact ) { BloomFilter filter = sent_to_bloom; if ( filter == null || filter.getEntryCount() > 100 ){ filter = BloomFilterFactory.createAddOnly(500); sent_to_bloom = filter; } filter.add( contact.getID()); } }
@Override public boolean contains( byte[] value ) { return( current_filter.contains(value)); }
protected void updateBloom( InetSocketAddress destination_address ) { // allow unresolved through (e.g. ipv6 dht seed) as handled later if ( !destination_address.isUnresolved()){ long diff = SystemTime.getCurrentTime() - last_bloom_rotation_time; if( diff < 0 || diff > BLOOM_ROTATION_PERIOD ) { // System.out.println( "bloom rotate: entries = " + bloom1.getEntryCount() + "/" + bloom2.getEntryCount()); bloom1 = bloom2; bloom2 = BloomFilterFactory.createAddOnly( BLOOM_FILTER_SIZE ); last_bloom_rotation_time = SystemTime.getCurrentTime(); } byte[] address_bytes = destination_address.getAddress().getAddress(); bloom1.add( address_bytes ); bloom2.add( address_bytes ); } }
int size = negative_result_bloom.getEntryCount(); if ( negative_result_bloom.contains( traversal.getTarget().toString().getBytes())){
@Override public int getEntryCount() { return( current_filter.getEntryCount()); }
@Override public String getString() { return( "ind=" + current_filter_index + ",filt=" + current_filter.getString()); } }
public BloomFilterRotator( BloomFilter _target, int _num ) { filters = new BloomFilter[_num]; filters[0] = _target; for (int i=1;i<filters.length;i++){ filters[i] = _target.getReplica(); } current_filter = _target; current_filter_index = 0; }
@Override public int getSize() { return( current_filter.getSize()); }
@Override public int count( byte[] value ) { return( current_filter.count( value )); }
@Override public void clear() { start_time = SystemTime.getMonotonousTime(); for ( BloomFilter filter: filters ){ filter.clear(); } }
protected void incrementValueAdds( DHTTransportContact contact ) { // assume a node stores 1000 values at 20 (K) locations -> 20,000 values // assume a DHT size of 100,000 nodes // that is, on average, 1 value per 5 nodes // assume NAT of up to 30 ports per address // this gives 6 values per address // with a factor of 10 error this is still only 60 per address // However, for CVS DHTs we can have sizes of 1000 or less. byte[] bloom_key = contact.getBloomKey(); int hit_count = ip_count_bloom_filter.add( bloom_key ); if ( DHTLog.GLOBAL_BLOOM_TRACE ){ System.out.println( "direct add from " + contact.getAddress() + ", hit count = " + hit_count ); } // allow up to 10% bloom filter utilisation if ( ip_count_bloom_filter.getSize() / ip_count_bloom_filter.getEntryCount() < 10 ){ rebuildIPBloomFilter( true ); } if ( hit_count > 64 ){ // obviously being spammed, drop all data originated by this IP and ban it banContact( contact, "global flood" ); } }