/** * Creates a new {@link BytesRefArray} with a counter to track allocated bytes */ public BytesRefArray(Counter bytesUsed) { this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator( bytesUsed)); pool.nextBuffer(); bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER * Integer.BYTES); this.bytesUsed = bytesUsed; }
/** * Resets the pool to its initial state reusing the first buffer and fills all * buffers with <tt>0</tt> bytes before they reused or passed to * {@link Allocator#recycleByteBlocks(byte[][], int, int)}. Calling * {@link ByteBlockPool#nextBuffer()} is not needed after reset. */ public void reset() { reset(true, true); }
/** Write byte into byte slice stream */ @Override public void writeByte(byte b) { assert slice != null; if (slice[upto] != 0) { upto = pool.allocSlice(slice, upto); slice = pool.buffer; offset0 = pool.byteOffset; assert slice != null; } slice[upto++] = b; assert upto != slice.length; }
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }
/** * Creates a new {@link BytesRefHash} with a {@link ByteBlockPool} using a * {@link DirectAllocator}. */ public BytesRefHash() { this(new ByteBlockPool(new DirectAllocator())); }
@Override protected BytesRef get(int i) { pool.setBytesRef(scratch, bytesStart[compact[i]]); return scratch; }
/** * Allocates a new slice with the given size. * @see ByteBlockPool#FIRST_LEVEL_SIZE */ public int newSlice(final int size) { if (byteUpto > BYTE_BLOCK_SIZE-size) nextBuffer(); final int upto = byteUpto; byteUpto += size; buffer[byteUpto-1] = 16; return upto; }
/** * Set the given {@link BytesRef} so that its content is equal to the * {@code ref.length} bytes starting at {@code offset}. Most of the time this * method will set pointers to internal data-structures. However, in case a * value crosses a boundary, a fresh copy will be returned. * On the contrary to {@link #setBytesRef(BytesRef, int)}, this does not * expect the length to be encoded with the data. */ public void setRawBytesRef(BytesRef ref, final long offset) { int bufferIndex = (int) (offset >> BYTE_BLOCK_SHIFT); int pos = (int) (offset & BYTE_BLOCK_MASK); if (pos + ref.length <= BYTE_BLOCK_SIZE) { ref.bytes = buffers[bufferIndex]; ref.offset = pos; } else { ref.bytes = new byte[ref.length]; ref.offset = 0; readBytes(offset, ref.bytes, 0, ref.length); } }
/** * Appends a copy of the given {@link BytesRef} to this {@link BytesRefArray}. * @param bytes the bytes to append * @return the index of the appended bytes */ @Override public int append(BytesRef bytes) { if (lastElement >= offsets.length) { int oldLen = offsets.length; offsets = ArrayUtil.grow(offsets, offsets.length + 1); bytesUsed.addAndGet((offsets.length - oldLen) * Integer.BYTES); } pool.append(bytes); offsets[lastElement++] = currentOffset; currentOffset += bytes.length; return lastElement-1; }
TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) { this.docState = docWriter.docState; this.trackAllocations = trackAllocations; this.nextTermsHash = nextTermsHash; this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter(); intPool = new IntBlockPool(docWriter.intBlockAllocator); bytePool = new ByteBlockPool(docWriter.byteBlockAllocator); if (nextTermsHash != null) { // We are primary termBytePool = bytePool; nextTermsHash.termBytePool = bytePool; } }
bytePool.nextBuffer(); final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset;
@Override public BytesRef next() { ord++; if (ord >= numTerms) { return null; } else { int textStart = postingsArray.textStarts[sortedTermIDs[ord]]; terms.bytePool.setBytesRef(scratch, textStart); return scratch; } }
/** * Creates a new byte slice with the given starting size and * returns the slices offset in the pool. */ public int allocSlice(final byte[] slice, final int upto) { final int level = slice[upto] & 15; final int newLevel = NEXT_LEVEL_ARRAY[level]; final int newSize = LEVEL_SIZE_ARRAY[newLevel]; // Maybe allocate another block if (byteUpto > BYTE_BLOCK_SIZE-newSize) { nextBuffer(); } final int newUpto = byteUpto; final int offset = newUpto + byteOffset; byteUpto += newSize; // Copy forward the past 3 bytes (which we are about // to overwrite with the forwarding address): buffer[newUpto] = slice[upto-3]; buffer[newUpto+1] = slice[upto-2]; buffer[newUpto+2] = slice[upto-1]; // Write forwarding address at end of last slice: slice[upto-3] = (byte) (offset >>> 24); slice[upto-2] = (byte) (offset >>> 16); slice[upto-1] = (byte) (offset >>> 8); slice[upto] = (byte) offset; // Write new level: buffer[byteUpto-1] = (byte) (16|newLevel); return newUpto+3; }
/** Fill the provided {@link BytesRef} with the bytes at the specified offset/length slice. * This will avoid copying the bytes, if the slice fits into a single block; otherwise, it uses * the provided {@link BytesRefBuilder} to copy bytes over. */ void setBytesRef(BytesRefBuilder builder, BytesRef result, long offset, int length) { result.length = length; int bufferIndex = (int) (offset >> BYTE_BLOCK_SHIFT); byte[] buffer = buffers[bufferIndex]; int pos = (int) (offset & BYTE_BLOCK_MASK); if (pos + length <= BYTE_BLOCK_SIZE) { // common case where the slice lives in a single block: just reference the buffer directly without copying result.bytes = buffer; result.offset = pos; } else { // uncommon case: the slice spans at least 2 blocks, so we must copy the bytes: builder.grow(length); result.bytes = builder.get().bytes; result.offset = 0; readBytes(offset, result.bytes, 0, length); } }
public void addPackedValue(int docID, BytesRef value) { if (value == null) { throw new IllegalArgumentException("field=" + fieldInfo.name + ": point value must not be null"); } if (value.length != packedBytesLength) { throw new IllegalArgumentException("field=" + fieldInfo.name + ": this field's value has length=" + value.length + " but should be " + (fieldInfo.getPointDataDimensionCount() * fieldInfo.getPointNumBytes())); } if (docIDs.length == numPoints) { docIDs = ArrayUtil.grow(docIDs, numPoints+1); iwBytesUsed.addAndGet((docIDs.length - numPoints) * Integer.BYTES); } bytes.append(value); docIDs[numPoints] = docID; if (docID != lastDocID) { numDocs++; lastDocID = docID; } numPoints++; }
/** * Creates a new {@link BytesRefList} */ public BytesRefList() { this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator( bytesUsed)); pool.nextBuffer(); bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_INT); }
public PointValuesWriter(DocumentsWriterPerThread docWriter, FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; this.iwBytesUsed = docWriter.bytesUsed; this.bytes = new ByteBlockPool(docWriter.byteBlockAllocator); docIDs = new int[16]; iwBytesUsed.addAndGet(16 * Integer.BYTES); packedBytesLength = fieldInfo.getPointDataDimensionCount() * fieldInfo.getPointNumBytes(); }
/** * Clears this {@link BytesRefArray} */ @Override public void clear() { lastElement = 0; currentOffset = 0; // TODO: it's trappy that this does not return storage held by int[] offsets array! Arrays.fill(offsets, 0); pool.reset(false, true); // no need to 0 fill the buffers we control the allocator }
public void add(int textStart) throws IOException { int termID = bytesHash.addByPoolOffset(textStart); if (termID >= 0) { // New posting // First time we are seeing this token since we last // flushed the hash. // Init stream slices if (numPostingInt + intPool.intUpto > IntBlockPool.INT_BLOCK_SIZE) { intPool.nextBuffer(); } if (ByteBlockPool.BYTE_BLOCK_SIZE - bytePool.byteUpto < numPostingInt*ByteBlockPool.FIRST_LEVEL_SIZE) { bytePool.nextBuffer(); } intUptos = intPool.buffer; intUptoStart = intPool.intUpto; intPool.intUpto += streamCount; postingsArray.intStarts[termID] = intUptoStart + intPool.intOffset; for(int i=0;i<streamCount;i++) { final int upto = bytePool.newSlice(ByteBlockPool.FIRST_LEVEL_SIZE); intUptos[intUptoStart+i] = upto + bytePool.byteOffset; } postingsArray.byteStarts[termID] = intUptos[intUptoStart]; newTerm(termID); } else { termID = (-termID)-1; int intStart = postingsArray.intStarts[termID]; intUptos = intPool.buffers[intStart >> IntBlockPool.INT_BLOCK_SHIFT]; intUptoStart = intStart & IntBlockPool.INT_BLOCK_MASK; addTerm(termID); } }
public void seekExact(long ord) { this.ord = (int) ord; int textStart = postingsArray.textStarts[sortedTermIDs[this.ord]]; terms.bytePool.setBytesRef(scratch, textStart); }