/** * Setup for reading the key of an entry with the equalKey method. * @param hashMapStore * @param refWord */ public void setKey(VectorMapJoinFastBytesHashMapStore hashMapStore, long refWord) { // Preconditions.checkState(!KeyRef.getIsInvalidFlag(refWord)); this.hashMapStore = hashMapStore; this.refWord = refWord; absoluteOffset = KeyRef.getAbsoluteOffset(refWord); // Position after next relative offset (fixed length) to the key. hashMapStore.writeBuffers.setReadPoint(absoluteOffset, readPos); keyLength = KeyRef.getSmallKeyLength(refWord); boolean isKeyLengthSmall = (keyLength != KeyRef.SmallKeyLength.allBitsOn); if (isKeyLengthSmall) { keyAbsoluteOffset = absoluteOffset; } else { // And, if current value is big we must read it. keyLength = hashMapStore.writeBuffers.readVInt(readPos); keyAbsoluteOffset = hashMapStore.writeBuffers.getReadPoint(readPos); } // NOTE: Reading is now positioned before the key bytes. }
public long add(byte[] keyBytes, int keyStart, int keyLength) { boolean isKeyLengthBig = (keyLength >= SmallKeyLength.threshold); long absoluteKeyOffset = writeBuffers.getWritePoint(); if (isKeyLengthBig) { writeBuffers.writeVInt(keyLength); } writeBuffers.write(keyBytes, keyStart, keyLength); long keyRefWord = IsNonZeroFlag.flagOnMask; if (isKeyLengthBig) { keyRefWord |= SmallKeyLength.allBitsOnBitShifted; } else { keyRefWord |= ((long) keyLength) << SmallKeyLength.bitShift; } keyRefWord |= absoluteKeyOffset; // LOG.debug("VectorMapJoinFastKeyStore add keyLength " + keyLength + " absoluteKeyOffset " + absoluteKeyOffset + " keyRefWord " + Long.toHexString(keyRefWord)); return keyRefWord; }
/** THIS METHOD IS NOT THREAD-SAFE. Use only at load time (or be mindful of thread safety). */ public int unsafeHashCode(long offset, int length) { return hashCode(offset, length, unsafeReadPos); }
public void getKey(long keyRefWord, ByteSegmentRef keyByteSegmentRef, WriteBuffers.Position readPos) { int storedKeyLength = (int) ((keyRefWord & SmallKeyLength.bitMask) >> SmallKeyLength.bitShift); boolean isKeyLengthSmall = (storedKeyLength != SmallKeyLength.allBitsOn); long absoluteKeyOffset = (keyRefWord & AbsoluteKeyOffset.bitMask); writeBuffers.setReadPoint(absoluteKeyOffset, readPos); if (!isKeyLengthSmall) { // Read big key length we wrote with the key. storedKeyLength = writeBuffers.readVInt(readPos); } writeBuffers.getByteSegmentRefToCurrent(keyByteSegmentRef, storedKeyLength, readPos); }
public long readNByteLong(long offset, int bytes, Position readPos) { setReadPoint(offset, readPos); long v = 0; if (isAllInOneReadBuffer(bytes, readPos)) { for (int i = 0; i < bytes; ++i) { v = (v << 8) + (readPos.buffer[readPos.offset + i] & 0xff); } readPos.offset += bytes; } else { for (int i = 0; i < bytes; ++i) { v = (v << 8) + (readNextByte(readPos) & 0xff); } } return v; }
/** * @param ref The ref. * @return The offset to list record pointer; list record is created if it doesn't exist. */ private long createOrGetListRecord(long ref) { if (Ref.hasList(ref)) { // LOG.info("Found list record at " + writeBuffers.getReadPoint()); return writeBuffers.getUnsafeReadPoint(); // Assumes we are here after key compare. } long firstTailOffset = Ref.getOffset(ref); // LOG.info("First tail offset to create list record is " + firstTailOffset); // Determine the length of storage for value and key lengths of the first record. writeBuffers.setUnsafeReadPoint(firstTailOffset); writeBuffers.unsafeSkipVLong(); writeBuffers.unsafeSkipVLong(); int lengthsLength = (int)(writeBuffers.getUnsafeReadPoint() - firstTailOffset); // Create the list record, copy first record value/key lengths there. writeBuffers.writeBytes(firstTailOffset, lengthsLength); long lrPtrOffset = writeBuffers.getWritePoint(); // LOG.info("Creating list record: copying " + lengthsLength + ", lrPtrOffset " + lrPtrOffset); // Reserve 5 bytes for writeValueRecord to fill. There might be junk there so null them. writeBuffers.write(FIVE_ZEROES); // Link the list record to the first element. writeBuffers.writeFiveByteULong(firstTailOffset, lrPtrOffset - lengthsLength - firstTailOffset); return lrPtrOffset; }
writeBuffers.setReadPoint(getFirstRecordLengthsOffset(ref, readPos), readPos); int valueLength = (int) writeBuffers.readVLong(readPos); int keyLength = (int) writeBuffers.readVLong(readPos); long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength); writeBuffers.populateValue(keyRef); long offsetAfterListRecordKeyLen = hasList ? writeBuffers.getReadPoint(readPos) : 0;
hashMap.writeBuffers.setReadPoint(firstOffset, readPos); int valueLength = (int) hashMap.writeBuffers.readVLong(readPos); hashMap.writeBuffers.populateValue(byteSegmentRef); long relativeOffset = hashMap.writeBuffers.readNByteLong(firstOffset, 5, readPos); hashMap.writeBuffers.setReadPoint(firstOffset + relativeOffset, readPos); int valueLength = (int) hashMap.writeBuffers.readVLong(readPos); hashMap.writeBuffers.populateValue(byteSegmentRef); nextTailOffset = hashMap.writeBuffers.readNByteLong(offsetAfterListRecordKeyLen, 5, readPos); if (nextTailOffset <= 0) { throw new Error("Expecting a second value"); hashMap.writeBuffers.setReadPoint(nextTailOffset, readPos); int valueLength = (int) hashMap.writeBuffers.readVLong(readPos); long delta = hashMap.writeBuffers.readVLong(readPos); long newTailOffset = delta == 0 ? 0 : (nextTailOffset - delta); hashMap.writeBuffers.populateValue(byteSegmentRef);
/** * @param ref The ref. * @return The offset to list record pointer; list record is created if it doesn't exist. */ private long createOrGetListRecord(long ref) { if (Ref.hasList(ref)) { // LOG.info("Found list record at " + writeBuffers.getReadPoint()); return writeBuffers.getReadPoint(); // Assumes we are here after key compare. } long firstTailOffset = Ref.getOffset(ref); // LOG.info("First tail offset to create list record is " + firstTailOffset); // Determine the length of storage for value and key lengths of the first record. writeBuffers.setReadPoint(firstTailOffset); writeBuffers.skipVLong(); writeBuffers.skipVLong(); int lengthsLength = (int)(writeBuffers.getReadPoint() - firstTailOffset); // Create the list record, copy first record value/key lengths there. writeBuffers.writeBytes(firstTailOffset, lengthsLength); long lrPtrOffset = writeBuffers.getWritePoint(); // LOG.info("Creating list record: copying " + lengthsLength + ", lrPtrOffset " + lrPtrOffset); // Reserve 5 bytes for writeValueRecord to fill. There might be junk there so null them. writeBuffers.write(FIVE_ZEROES); // Link the list record to the first element. writeBuffers.writeFiveByteULong(firstTailOffset, lrPtrOffset - lengthsLength - firstTailOffset); return lrPtrOffset; }
long nextAbsoluteValueOffset = writeBuffers.getWritePoint(); writeBuffers.write(RelativeOffset.zeroPadding); } else { writeBuffers.readNByteLong( referenceAbsoluteOffset, RelativeOffset.byteLength, unsafeReadPos); writeBuffers.writeFiveByteULong(newPrevRelativeValueOffset); writeBuffers.writeVInt(valueLength); writeBuffers.write(valueBytes, valueStart, valueLength); writeBuffers.writeFiveByteULong(referenceAbsoluteOffset, newRelativeOffset);
writeBuffers.write(FOUR_ZEROES); long keyOffset = writeBuffers.getWritePoint(); int keyLength = (int)(writeBuffers.getWritePoint() - keyOffset); int hashCode = (keyHashCode == -1) ? writeBuffers.unsafeHashCode(keyOffset, keyLength) : keyHashCode; } else { writeBuffers.setWritePoint(keyOffset - 4); long lrPtrOffset = createOrGetListRecord(ref); long tailOffset = writeValueAndLength(kv);
@Override public void writeInt(long offset, int v) { int prevIndex = writePos.bufferIndex, prevOffset = writePos.offset; setWritePoint(offset); if (isAllInOneWriteBuffer(4)) { writePos.buffer[writePos.offset] = (byte)(v >> 24); writePos.buffer[writePos.offset + 1] = (byte)(v >> 16); writePos.buffer[writePos.offset + 2] = (byte)(v >> 8); writePos.buffer[writePos.offset + 3] = (byte)(v); writePos.offset += 4; } else { setByte(offset++, (byte)(v >>> 24)); setByte(offset++, (byte)(v >>> 16)); setByte(offset++, (byte)(v >>> 8)); setByte(offset, (byte)(v)); } writePos.bufferIndex = prevIndex; writePos.buffer = writeBuffers.get(writePos.bufferIndex); writePos.offset = prevOffset; }
valueStore.writeBuffers.setReadPoint(absoluteValueOffset, readPos); valueLength = valueStore.writeBuffers.readVInt(readPos); } else { long relativeOffsetWord = valueStore.writeBuffers.readVLong(readPos); valueLength = valueStore.writeBuffers.readVInt(readPos); nextSmallValueLength = valueStore.writeBuffers.readVInt(readPos); } else { nextSmallValueLength = -1; valueStore.writeBuffers.setReadPoint(absoluteValueOffset, readPos); valueLength = valueStore.writeBuffers.readVInt(readPos); long relativeOffsetWord = valueStore.writeBuffers.readVLong(readPos); valueLength = nextSmallValueLength; } else { valueLength = valueStore.writeBuffers.readVInt(readPos); nextSmallValueLength = valueStore.writeBuffers.readVInt(readPos); } else { nextSmallValueLength = -1; valueStore.writeBuffers.getByteSegmentRefToCurrent(byteSegmentRef, valueLength, readPos);
writeBuffers.write(FOUR_ZEROES); long keyOffset = writeBuffers.getWritePoint(); int keyLength = (int)(writeBuffers.getWritePoint() - keyOffset); int hashCode = (keyHashCode == -1) ? writeBuffers.hashCode(keyOffset, keyLength) : keyHashCode; } else { writeBuffers.setWritePoint(keyOffset - 4); long lrPtrOffset = createOrGetListRecord(ref); long tailOffset = writeValueAndLength(kv);
hashMapStore.writeBuffers.setReadPoint( referenceAbsoluteOffset, readPos); long relativeNextValueOffset = hashMapStore.writeBuffers.readNByteLong( KeyRef.AbsoluteOffset.byteLength, readPos); hashMapStore.writeBuffers.setReadPoint(firstValueAbsoluteOffset, readPos); nextValueLength = hashMapStore.writeBuffers.readVInt(readPos); } else { hashMapStore.writeBuffers.setReadPoint(nextAbsoluteValueOffset, readPos); hashMapStore.writeBuffers.readNByteLong( RelativeOffset.byteLength, readPos); if (relativeNextValueOffset == 0) { nextValueLength = hashMapStore.writeBuffers.readVInt(readPos); hashMapStore.writeBuffers.getByteSegmentRefToCurrent( byteSegmentRef, nextValueLength, readPos);
/** * Same as {@link #isSameKey(long, int, long, int)} but for externally stored key. */ private boolean isSameKey(byte[] key, int offset, int length, long ref, int hashCode, WriteBuffers.Position readPos) { if (!compareHashBits(ref, hashCode)) { return false; // Hash bits don't match. } writeBuffers.setReadPoint(getFirstRecordLengthsOffset(ref, readPos), readPos); int valueLength = (int)writeBuffers.readVLong(readPos), keyLength = (int)writeBuffers.readVLong(readPos); long keyOffset = Ref.getOffset(ref) - (valueLength + keyLength); // See the comment in the other isSameKey if (offset == 0) { return writeBuffers.isEqual(key, length, keyOffset, keyLength); } else { return writeBuffers.isEqual(key, offset, length, keyOffset, keyLength); } }
/** * @param refWord */ public long bumpCount(long refWord, WriteBuffers.Position unsafeReadPos) { // Preconditions.checkState(!KeyRef.getIsInvalidFlag(refWord)); /* * Extract information from the reference word. */ final long countAbsoluteOffset = KeyRef.getAbsoluteOffset(refWord) - MultiSetCount.byteLength; final int currentCount = writeBuffers.readInt( countAbsoluteOffset, unsafeReadPos); // Mark reference as having more than 1 as the count. refWord &= KeyRef.IsSingleFlag.flagOffMask; // Save current write position. final long saveAbsoluteOffset = writeBuffers.getWritePoint(); writeBuffers.setWritePoint(countAbsoluteOffset); writeBuffers.writeInt( countAbsoluteOffset, currentCount + 1); // Restore current write position. writeBuffers.setWritePoint(saveAbsoluteOffset); return refWord; }
long valueOffset = writeBuffers.getWritePoint(); kv.writeValue(writeBuffers); long tailOffset = writeBuffers.getWritePoint(); int valueLength = (int)(tailOffset - valueOffset); writeBuffers.reserve(1); ++tailOffset; writeBuffers.writeVLong(valueLength); writeBuffers.writeVLong(keyLength); long lengthsLength = writeBuffers.getWritePoint() - tailOffset; if (lengthsLength < 5) { // Reserve space for potential future list writeBuffers.reserve(5 - (int)lengthsLength); writeBuffers.writeInt(keyOffset - 4, hashCode); return tailOffset;
/** * Take the segment reference from {@link #getValueRefs(byte[], int, List)} * result and makes it self-contained - adds byte array where the value is stored, and * updates the offset from "global" write buffers offset to offset within that array. */ public void populateValue(WriteBuffers.ByteSegmentRef valueRef) { writeBuffers.populateValue(valueRef); }
/** THIS METHOD IS NOT THREAD-SAFE. Use only at load time (or be mindful of thread safety). */ public long getUnsafeReadPoint() { return getReadPoint(unsafeReadPos); }