public boolean contains(BucketId id) { if (id.getUsedBits() < getUsedBits()) { return false; } BucketId copy = new BucketId(getUsedBits(), id.getRawId()); return (copy.getId() == getId()); }
if (!progress.equals(NULL_BUCKET) && !progress.equals(FINISHED_BUCKET) && !superbucket.contains(progress) && !progress.contains(superbucket)) { if (log.isLoggable(LogLevel.DEBUG)) { log.log(LogLevel.DEBUG, "updateProgress called with non-contained bucket " if (!progress.equals(FINISHED_BUCKET)) { if (entry.getState() != BucketState.BUCKET_ACTIVE) { if (log.isLoggable(LogLevel.DEBUG)) {
private int getStorageSeed(BucketId bucket, ClusterState state) { int seed = ((int) bucket.getRawId()) & distributionBitMasks[state.getDistributionBitCount()]; if (bucket.getUsedBits() > 33) { int usedBits = bucket.getUsedBits() - 1; seed ^= (distributionBitMasks[usedBits - 32] & (bucket.getRawId() >> 32)) << 6; } return seed; }
/** * @param superbucket The superbucket of which <code>progress</code> is * a sub-bucket * @param progress The sub-bucket for which a fractional progress should * be calculated * @return a value in [0, 1] specifying how far the (sub-bucket) has * reached in its superbucket. This is calculated by looking at the * bucket's split factor. */ public synchronized double progressFraction(BucketId superbucket, BucketId progress) { long revBits = bucketToKey(progress.getId()); int superUsed = superbucket.getUsedBits(); int progressUsed = progress.getUsedBits(); if (progressUsed == 0 || progressUsed < superUsed) { return 0; } int splitCount = progressUsed - superUsed; if (splitCount == 0) return 1; // Superbucket or inconsistent used-bits // Extract reversed split-bits revBits <<= superUsed; revBits >>>= 64 - splitCount; return (double)(revBits + 1) / (double)(1L << splitCount); }
public CreateVisitorReply(int type) { super(type); lastBucket = new BucketId(Integer.MAX_VALUE); }
@Override public long getSequenceId() { return bucket.getRawId(); }
int splitDistBits = bucket.getUsedBits() + 1; BucketId splitLeft = new BucketId(splitDistBits, bucket.getId()); BucketId splitRight = new BucketId(splitDistBits, bucket.getId() | (1L << bucket.getUsedBits()));
: buckets.entrySet()) { assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING); BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey())); if (pending.getUsedBits() < targetDistBits) { if (pending.getUsedBits() + 1 < targetDistBits) { maybeInconsistent = true; // Do another pass : buckets.entrySet()) { assert(entry.getValue().getState() == ProgressToken.BucketState.BUCKET_PENDING); BucketId pending = new BucketId(ProgressToken.keyToBucketId(entry.getKey().getKey())); if (pending.getUsedBits() > targetDistBits) { BucketId rightCheck = new BucketId(lastMergedBucket.getUsedBits(), lastMergedBucket.getId() | (1L << (lastMergedBucket.getUsedBits() - 1))); if (pending.equals(rightCheck)) { if (log.isLoggable(LogLevel.SPAM)) { log.log(LogLevel.SPAM, "Skipped " + pending + if (pending.getUsedBits() - 1 > targetDistBits) { maybeInconsistent = true; // Do another pass
public long getId() { int notUsed = 64 - getUsedBits(); long usedMask = (0xFFFFFFFFFFFFFFFFl << notUsed) >>> notUsed; long countMask = (0xFFFFFFFFFFFFFFFFl >>> (64 - COUNT_BITS)) << (64 - COUNT_BITS); return id & (usedMask | countMask); }
progressToken.updateProgress(superbucket, progress); if (superbucket.getUsedBits() != distributionBitCount) { if (!progress.equals(ProgressToken.FINISHED_BUCKET)) { log.log(LogLevel.DEBUG, "Received non-finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + "until all active are done"); log.log(LogLevel.DEBUG, "Received finished bucket " + superbucket + " with wrong distribution bit count (" + superbucket.getUsedBits() + "). Waiting to correct " + "until all active are done");
public boolean contains(DocumentId docId, BucketIdFactory factory) { return contains(factory.getBucketId(docId)); } }
@Override public boolean equals(Object obj) { if (!(obj instanceof BucketInfo)) { return false; } BucketInfo rhs = (BucketInfo)obj; if (bucket == null) { if (rhs.bucket != null) { return false; } } else if (!bucket.equals(rhs.bucket)) { return false; } if (bucketInformation == null) { if (rhs.bucketInformation != null) { return false; } } else if (!bucketInformation.equals(rhs.bucketInformation)) { return false; } return true; }
@Override public String toString() { StringBuilder sb = new StringBuilder().append("VisitorInfoMessage("); if (finishedBuckets.size() == 0) { sb.append("No buckets"); } else if (finishedBuckets.size() == 1) { sb.append("Bucket ").append(finishedBuckets.iterator().next().toString()); } else if (finishedBuckets.size() < 65536) { sb.append(finishedBuckets.size()).append(" buckets:"); Iterator<BucketId> it = finishedBuckets.iterator(); for (int i = 0; it.hasNext() && i < 3; ++i) { sb.append(' ').append(it.next().toString()); } if (it.hasNext()) { sb.append(" ..."); } } else { sb.append("All buckets"); } sb.append(", error message '").append(errorMessage).append('\''); return sb.append(')').toString(); } }
public BucketId toBucketId() { /* * Explanation time: since Java was designed so mankind could suffer, * shift ops on bytes have an implicit int conversion with sign-extend. * When a byte is negative, you end up with an int/long with a 0xFFFFFF * prefix, in turn causing your other friendly bitwise ORs to act * pretty far from what was originally intended. * To get around this, we explicitly sign extend before the compiler can * do so for us and make sure to OR away any sign extensions. */ long location = ((long)raw[0] & 0xFF) | (((long)raw[1] & 0xFF) << 8) | (((long)raw[2] & 0xFF) << 16) | (((long)raw[3] & 0xFF) << 24); long md5 = 0; for (int i = 4, j = 0; i < LENGTH; i++, j += 8) { md5 |= ((long)raw[i] & 0xFF) << j; } // Drumroll: this is why 'location' is of type long. Otherwise, the // ORing would sign-extend it and cause havoc when its MSB is set. long rawBucketId = (md5 & 0xFFFFFFFF00000000L) | location; return new BucketId(58, rawBucketId); }
private int getDistributorSeed(BucketId bucket, ClusterState state) { return ((int) bucket.getRawId()) & distributionBitMasks[state.getDistributionBitCount()]; }
int usedBits = bucket.getUsedBits(); if ((bucket.getId() & (1L << (usedBits - 1))) == 0) { BucketId rightCheck = new BucketId(usedBits, bucket.getId() | (1L << (usedBits - 1))); BucketEntry rightSibling = buckets.get(bucketToKeyWrapper(rightCheck)); if (rightSibling.getProgress().getUsedBits() != 0 && log.isLoggable(LogLevel.DEBUG)) { log.log(LogLevel.DEBUG, "Bucket progress for " + rightCheck + BucketId leftSanityCheck = new BucketId(usedBits, bucket.getId() & ~(1L << (usedBits - 1))); BucketEntry leftSibling = buckets.get(bucketToKeyWrapper(leftSanityCheck)); assert(leftSibling == null) : "bucket merge sanity checking failed"; BucketId newMerged = new BucketId(usedBits - 1, bucket.getId()); addBucket(newMerged, entry.getProgress(), BucketState.BUCKET_PENDING);
public String toString() { StringBuilder sb = new StringBuilder().append("BucketId(0x"); String number = Long.toHexString(getId()); for (int i=number.length(); i<16; ++i) { sb.append('0'); } sb.append(number).append(')'); return sb.toString(); }