org.apache.solr.common.util.Hash java code examples

StringBytes reset(byte[] bytes, int offset, int length) {
 this.bytes = bytes;
 this.offset = offset;
 this.length = length;
 hash = bytes == null ? 0 : Hash.murmurhash3_x86_32(bytes, offset, length, 0);
 return this;
}

/**
 * Identical to lookup3, except initval is biased by -(length&lt;&lt;2).
 * This is equivalent to leaving out the length factor in the initial state.
 * {@code lookup3ycs(k,offset,length,initval) == lookup3(k,offset,length,initval-(length<<2))}
 * and
 * {@code lookup3ycs(k,offset,length,initval+(length<<2)) == lookup3(k,offset,length,initval)}
 */
public static int lookup3ycs(int[] k, int offset, int length, int initval) {
 return lookup3(k, offset, length, initval-(length<<2));
}

public void add(String content) {
 hash = Hash.lookup3ycs64(content,0,content.length(),hash);
}

 long k1 = getLongLittleEndian(key, i);
 long k2 = getLongLittleEndian(key, i+8);
 k1 *= c1; k1  = Long.rotateLeft(k1,31); k1 *= c2; h1 ^= k1;
 h1 = Long.rotateLeft(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
h2 += h1;
h1 = fmix64(h1);
h2 = fmix64(h2);

 long k1 = getLongLittleEndian(key, i);
 long k2 = getLongLittleEndian(key, i+8);
 k1 *= c1; k1  = Long.rotateLeft(k1,31); k1 *= c2; h1 ^= k1;
 h1 = Long.rotateLeft(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
h2 += h1;
h1 = fmix64(h1);
h2 = fmix64(h2);

StringBytes reset(byte[] bytes, int offset, int length) {
 this.bytes = bytes;
 this.offset = offset;
 this.length = length;
 hash = bytes == null ? 0 : Hash.murmurhash3_x86_32(bytes, offset, length, 0);
 return this;
}

/**
 * Identical to lookup3, except initval is biased by -(length&lt;&lt;2).
 * This is equivalent to leaving out the length factor in the initial state.
 * {@code lookup3ycs(k,offset,length,initval) == lookup3(k,offset,length,initval-(length<<2))}
 * and
 * {@code lookup3ycs(k,offset,length,initval+(length<<2)) == lookup3(k,offset,length,initval)}
 */
public static int lookup3ycs(int[] k, int offset, int length, int initval) {
 return lookup3(k, offset, length, initval-(length<<2));
}

public int sliceHash(String id, SolrInputDocument sdoc, SolrParams params, DocCollection collection) {
 return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
}

public int sliceHash(String id, SolrInputDocument sdoc, SolrParams params, DocCollection collection) {
 return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
}

 hashes[i] = Hash.murmurhash3_x86_32("", 0, "".length(), 0);
else
 hashes[i] = Hash.murmurhash3_x86_32(parts[i], 0, parts[i].length(), 0);

 hashes[i] = Hash.murmurhash3_x86_32("", 0, "".length(), 0);
else
 hashes[i] = Hash.murmurhash3_x86_32(parts[i], 0, parts[i].length(), 0);

@Override
public int sliceHash(String id, SolrInputDocument doc, SolrParams params, DocCollection collection) {
 String shardFieldName = getRouteField(collection);
 if (shardFieldName != null && doc != null) {
  Object o = doc.getFieldValue(shardFieldName);
  if (o == null)
   throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No value for :" + shardFieldName + ". Unable to identify shard");
  id = o.toString();
 }
 if (id.indexOf(SEPARATOR) < 0) {
  return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
 }
 return new KeyParser(id).getHash();
}

int hashCode = Hash.murmurhash3_x86_32(keyStr, 0, keyStr.length(), 0); 
int offset = (hashCode & Integer.MAX_VALUE) % (numPartitions / shards);
int microShard = (rootShard * (numPartitions / shards)) + offset;

@Override
public int sliceHash(String id, SolrInputDocument doc, SolrParams params, DocCollection collection) {
 String shardFieldName = getRouteField(collection);
 if (shardFieldName != null && doc != null) {
  Object o = doc.getFieldValue(shardFieldName);
  if (o == null)
   throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No value for :" + shardFieldName + ". Unable to identify shard");
  id = o.toString();
 }
 if (id.indexOf(SEPARATOR) < 0) {
  return Hash.murmurhash3_x86_32(id, 0, id.length(), 0);
 }
 return new KeyParser(id).getHash();
}

int hashCode = Hash.murmurhash3_x86_32(keyStr, 0, keyStr.length(), 0); 
int offset = (hashCode & Integer.MAX_VALUE) % (numPartitions / shards);
int microShard = (rootShard * (numPartitions / shards)) + offset;

@Override
public Collection<Slice> getSearchSlicesSingle(String shardKey, SolrParams params, DocCollection collection) {
 if (shardKey == null) {
  // search across whole collection
  // TODO: this may need modification in the future when shard splitting could cause an overlap
  return collection.getActiveSlices();
 }
 String id = shardKey;
 if (shardKey.indexOf(SEPARATOR) < 0) {
  // shardKey is a simple id, so don't do a range
  return Collections.singletonList(hashToSlice(Hash.murmurhash3_x86_32(id, 0, id.length(), 0), collection));
 }
 Range completeRange = new KeyParser(id).getRange();
 List<Slice> targetSlices = new ArrayList<>(1);
 for (Slice slice : collection.getActiveSlicesArr()) {
  Range range = slice.getRange();
  if (range != null && range.overlaps(completeRange)) {
   targetSlices.add(slice);
  }
 }
 return targetSlices;
}

@Override
public Collection<Slice> getSearchSlicesSingle(String shardKey, SolrParams params, DocCollection collection) {
 if (shardKey == null) {
  // search across whole collection
  // TODO: this may need modification in the future when shard splitting could cause an overlap
  return collection.getActiveSlices();
 }
 String id = shardKey;
 if (shardKey.indexOf(SEPARATOR) < 0) {
  // shardKey is a simple id, so don't do a range
  return Collections.singletonList(hashToSlice(Hash.murmurhash3_x86_32(id, 0, id.length(), 0), collection));
 }
 Range completeRange = new KeyParser(id).getRange();
 List<Slice> targetSlices = new ArrayList<>(1);
 for (Slice slice : collection.getActiveSlices()) {
  Range range = slice.getRange();
  if (range != null && range.overlaps(completeRange)) {
   targetSlices.add(slice);
  }
 }
 return targetSlices;
}

final Object lock = locks.get(Math.abs(Hash.murmurhash3_x86_32(collection, 0, collection.length(), 0) % locks.size()));
synchronized (lock) {

final Object lock = locks.get(Math.abs(Hash.murmurhash3_x86_32(collection, 0, collection.length(), 0) % locks.size()));
DocCollection fetchedCol = null;
synchronized (lock) {

Javadoc

Fast, well distributed, cross-platform hash functions.

Development background: I was surprised to discovered that there isn't a good cross-platform hash function defined for strings. MD5, SHA, FVN, etc, all define hash functions over bytes, meaning that it's under-specified for strings.

So I set out to create a standard 32 bit string hash that would be well defined for implementation in all languages, have very high performance, and have very good hash properties such as distribution. After evaluating all the options, I settled on using Bob Jenkins' lookup3 as a base. It's a well studied and very fast hash function, and the hashword variant can work with 32 bits at a time (perfect for hashing unicode code points). It's also even faster on the latest JVMs which can translate pairs of shifts into native rotate instructions.

The only problem with using lookup3 hashword is that it includes a length in the initial value. This would suck some performance out since directly hashing a UTF8 or UTF16 string (Java) would require a pre-scan to get the actual number of unicode code points. The solution was to simply remove the length factor, which is equivalent to biasing initVal by -(numCodePoints*4). This slightly modified lookup3 I define as lookup3ycs.

So the definition of the cross-platform string hash lookup3ycs is as follows:

The hash value of a character sequence (a string) is defined to be the hash of its unicode code points, according to lookup3 hashword, with the initval biased by -(length*4).

So by definition

 
lookup3ycs(k,offset,length,initval) == lookup3(k,offset,length,initval-(length*4)) 
AND 
lookup3ycs(k,offset,length,initval+(length*4)) == lookup3(k,offset,length,initval)

An obvious advantage of this relationship is that you can use lookup3 if you don't have an implementation of lookup3ycs.

Most used methods

murmurhash3_x86_32
Returns the MurmurHash3_x86_32 hash. Original source/tests at https://github.com/yonik/java_util/
fmix64
getLongLittleEndian
Gets a long from a byte buffer in little endian byte order.
lookup3
A Java implementation of hashword from lookup3.c by Bob Jenkins (original source [http://burtleburtl
lookup3ycs64
This is the 64 bit version of lookup3ycs, corresponding to Bob Jenkin's lookup3 hashlittle2 with ini

Popular in Java

Updating database using SQL prepared statement
getResourceAsStream (ClassLoader)
getSystemService (Context)
scheduleAtFixedRate (Timer)
ArrayList (java.util)
ArrayList is an implementation of List, backed by an array. All optional operations including adding
Dictionary (java.util)
Note: Do not use this class since it is obsolete. Please use the Map interface for new implementatio
LinkedList (java.util)
Doubly-linked list implementation of the List and Dequeinterfaces. Implements all optional list oper
TreeMap (java.util)
Walk the nodes of the tree left-to-right or right-to-left. Note that in descending iterations, next
Annotation (javassist.bytecode.annotation)
The annotation structure.An instance of this class is returned bygetAnnotations() in AnnotationsAttr
BasicDataSource (org.apache.commons.dbcp)
Basic implementation of javax.sql.DataSource that is configured via JavaBeans properties. This is no
Github Copilot alternatives

How to useHash in org.apache.solr.common.util

Best Java code snippets using org.apache.solr.common.util.Hash (Showing top 19 results out of 315)

How to use
Hash
in
org.apache.solr.common.util