Refine search
public void setFloorData(ByteArrayDataInput in, BytesRef source) { final int numBytes = source.length - (in.getPosition() - source.offset); if (numBytes > floorData.length) { floorData = new byte[ArrayUtil.oversize(numBytes, 1)]; } System.arraycopy(source.bytes, source.offset+in.getPosition(), floorData, 0, numBytes); floorDataReader.reset(floorData, 0, numBytes); numFollowFloorBlocks = floorDataReader.readVInt(); nextFloorLabel = floorDataReader.readByte() & 0xff; //if (DEBUG) { //System.out.println(" setFloorData fpOrig=" + fpOrig + " bytes=" + new BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks=" + numFollowFloorBlocks + " nextFloorLabel=" + toHex(nextFloorLabel)); //} }
public PackedIndexTree() { int treeDepth = getTreeDepth(); leafBlockFPStack = new long[treeDepth+1]; leftNodePositions = new int[treeDepth+1]; rightNodePositions = new int[treeDepth+1]; splitValuesStack = new byte[treeDepth+1][]; splitDims = new int[treeDepth+1]; negativeDeltas = new boolean[numIndexDims*(treeDepth+1)]; in = new ByteArrayDataInput(packedIndex); splitValuesStack[0] = new byte[packedIndexBytesLength]; readNodeData(false); scratch = new BytesRef(); scratch.length = bytesPerDim; }
public void nextLeaf() { assert nextEnt != -1 && nextEnt < entCount: "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp; nextEnt++; suffix = suffixesReader.readVInt(); startBytePos = suffixesReader.getPosition(); suffixesReader.skipBytes(suffix); }
public boolean nextNonLeaf() { assert nextEnt != -1 && nextEnt < entCount: "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp; nextEnt++; final int code = suffixesReader.readVInt(); suffix = code >>> 1; startBytePos = suffixesReader.getPosition(); suffixesReader.skipBytes(suffix); if ((code & 1) == 0) { // A normal term termState.termBlockOrd++; return false; } else { // A sub-block; make sub-FP absolute: lastSubFP = fp - suffixesReader.readVLong(); return true; } }
long getMinLeafBlockFP() { if (packedIndex != null) { return new ByteArrayDataInput(packedIndex).readVLong(); } else { long minFP = Long.MAX_VALUE; for(long fp : leafBlockFPs) { minFP = Math.min(minFP, fp); } return minFP; } }
/** decodes the payload at the current position */ protected BytesRef decodePayload(BytesRef scratch, ByteArrayDataInput tmpInput) { tmpInput.reset(scratch.bytes); tmpInput.skipBytes(scratch.length - 2); // skip to payload size short payloadLength = tmpInput.readShort(); // read payload size tmpInput.setPosition(scratch.length - 2 - payloadLength); // setPosition to start of payload BytesRef payloadScratch = new BytesRef(payloadLength); tmpInput.readBytes(payloadScratch.bytes, 0, payloadLength); // read payload payloadScratch.length = payloadLength; scratch.length -= 2; // payload length info (short) scratch.length -= payloadLength; // payload return payloadScratch; } }
BytesRef surface = new BytesRef(); IntsRefBuilder scratchInts = new IntsRefBuilder(); ByteArrayDataInput input = new ByteArrayDataInput(); break; input.reset(bytes.bytes, bytes.offset, bytes.length); short analyzedLength = input.readShort(); analyzed.grow(analyzedLength+2); input.readBytes(analyzed.bytes(), 0, analyzedLength); analyzed.setLength(analyzedLength); long cost = input.readInt(); surface.length = input.readShort(); surface.offset = input.getPosition(); } else { surface.offset = input.getPosition(); surface.length = bytes.length - surface.offset; builder.add(scratchInts.get(), outputs.newPair(cost, BytesRef.deepCopyOf(surface))); } else { int payloadOffset = input.getPosition() + surface.length; int payloadLength = bytes.length - payloadOffset; BytesRef br = new BytesRef(surface.length + 1 + payloadLength);
readerA.reset(a.bytes, a.offset, a.length); scratchA.length = readerA.readShort(); scratchA.bytes = a.bytes; scratchA.offset = readerA.getPosition(); readerB.reset(b.bytes, b.offset, b.length); scratchB.bytes = b.bytes; scratchB.length = readerB.readShort(); scratchB.offset = readerB.getPosition(); readerA.skipBytes(scratchA.length); readerB.skipBytes(scratchB.length); long aCost = readerA.readInt(); long bCost = readerB.readInt(); if (aCost < bCost) { return -1; scratchA.length = readerA.readShort(); scratchA.offset = readerA.getPosition(); scratchB.length = readerB.readShort(); scratchB.offset = readerB.getPosition(); } else { scratchA.offset = readerA.getPosition(); scratchA.length = a.length - scratchA.offset; scratchB.offset = readerB.getPosition(); scratchB.length = b.length - scratchB.offset;
ByteArrayDataInput in = new ByteArrayDataInput(updates); BytesRef term = new BytesRef(); term.bytes = new byte[16]; BytesRef scratch = new BytesRef(); scratch.bytes = new byte[16]; while (in.getPosition() != updates.length) { int code = in.readVInt(); int docIDUpto = in.readVInt(); term.length = code >> 3; termField = in.readString(); updateField = in.readString(); term.bytes = ArrayUtil.grow(term.bytes, term.length); in.readBytes(term.bytes, 0, term.length);
SegmentTermsEnumFrame f = getFrame(ord); assert f != null; final BytesRef prefix = new BytesRef(term.get().bytes, 0, f.prefix); if (f.nextEnt == -1) { out.println(" frame " + (isSeekFrame ? "(seek)" : "(next)") + " ord=" + ord + " fp=" + f.fp + (f.isFloor ? (" (fpOrig=" + f.fpOrig + ")") : "") + " prefixLen=" + f.prefix + " prefix=" + prefix + (f.nextEnt == -1 ? "" : (" (of " + f.entCount + ")")) + " hasTerms=" + f.hasTerms + " isFloor=" + f.isFloor + " code=" + ((f.fp<< BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS) + (f.hasTerms ? BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS:0) + (f.isFloor ? BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR:0)) + " isLastInFloor=" + f.isLastInFloor + " mdUpto=" + f.metaDataUpto + " tbOrd=" + f.getTermBlockOrd()); throw new RuntimeException("seek state is broken"); } else if (isSeekFrame && !f.isFloor) { final ByteArrayDataInput reader = new ByteArrayDataInput(output.bytes, output.offset, output.length); final long codeOrig = reader.readVLong(); final long code = (f.fp << BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS) | (f.hasTerms ? BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS:0) | (f.isFloor ? BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR:0); if (codeOrig != code) {
@Override public int compare(BytesRef a, BytesRef b) { reader.reset(a.bytes, a.offset, a.length); final int latAEnc = reader.readInt(); final int lonAEnc = reader.readInt(); final int docIDA = reader.readVInt(); final long ordA = reader.readVLong(); reader.reset(b.bytes, b.offset, b.length); final int latBEnc = reader.readInt(); final int lonBEnc = reader.readInt(); final int docIDB = reader.readVInt(); final long ordB = reader.readVLong(); int cmp; if (lon) { cmp = Integer.compare(lonAEnc, lonBEnc); } else { cmp = Integer.compare(latAEnc, latBEnc); } if (cmp != 0) { return cmp; } // Tie-break cmp = Integer.compare(docIDA, docIDB); if (cmp != 0) { return cmp; } return Long.compare(ordA, ordB); } };
public void nextLeaf() { //if (DEBUG) System.out.println(" frame.next ord=" + ord + " nextEnt=" + nextEnt + " entCount=" + entCount); assert nextEnt != -1 && nextEnt < entCount: "nextEnt=" + nextEnt + " entCount=" + entCount + " fp=" + fp; nextEnt++; suffix = suffixesReader.readVInt(); startBytePos = suffixesReader.getPosition(); ste.term.setLength(prefix + suffix); ste.term.grow(ste.term.length()); suffixesReader.readBytes(ste.term.bytes(), prefix, suffix); ste.termExists = true; }
@Override public BytesRef nextValue() throws IOException { scratch.length = in.readVInt(); scratch.offset = in.getPosition(); in.setPosition(scratch.offset + scratch.length); return scratch; }
leafBlockFPStack[level] += in.readVLong(); int code = in.readVInt(); splitDim = code % numIndexDims; splitDims[level] = splitDim; in.readBytes(splitValuesStack[level], splitDim*bytesPerDim+prefix+1, suffix-1); } else { leftNumBytes = in.readVInt(); } else { leftNumBytes = 0; leftNodePositions[level] = in.getPosition(); rightNodePositions[level] = leftNodePositions[level] + leftNumBytes;
void load(BytesRef frameIndexData) throws IOException { if (frameIndexData != null) { floorDataReader.reset(frameIndexData.bytes, frameIndexData.offset, frameIndexData.length); final long code = floorDataReader.readVLong(); if ((code & BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0) { numFollowFloorBlocks = floorDataReader.readVInt(); nextFloorLabel = floorDataReader.readByte() & 0xff; fp = fpOrig + (floorDataReader.readVLong() >>> 1); numFollowFloorBlocks--; if (numFollowFloorBlocks != 0) { nextFloorLabel = floorDataReader.readByte() & 0xff; } else { nextFloorLabel = 256; suffixesReader.reset(suffixBytes, 0, numBytes); statsReader.reset(statBytes, 0, numBytes); metaDataUpto = 0; bytesReader.reset(bytes, 0, numBytes);
@Override public boolean advanceExact(int doc) throws IOException { if (values.advanceExact(doc)) { final BytesRef bytes = values.binaryValue(); assert bytes.length > 0; in.reset(bytes.bytes, bytes.offset, bytes.length); count = in.readVInt(); scratch.bytes = bytes.bytes; return true; } else { return false; } }
static ParsedScrollId parseScrollId(String scrollId) { try { byte[] bytes = Base64.getUrlDecoder().decode(scrollId); ByteArrayDataInput in = new ByteArrayDataInput(bytes); String type = in.readString(); ScrollIdForNode[] context = new ScrollIdForNode[in.readVInt()]; for (int i = 0; i < context.length; ++i) { long id = in.readLong(); String target = in.readString(); String clusterAlias; final int index = target.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR); if (index == -1) { clusterAlias = null; } else { clusterAlias = target.substring(0, index); target = target.substring(index+1); } context[i] = new ScrollIdForNode(clusterAlias, target, id); } if (in.getPosition() != bytes.length) { throw new IllegalArgumentException("Not all bytes were read"); } return new ParsedScrollId(scrollId, type, context); } catch (Exception e) { throw new IllegalArgumentException("Cannot parse scroll id", e); } }
final int savePos = currentFrame.suffixesReader.getPosition(); final int saveStartBytePos = currentFrame.startBytePos; final int saveSuffix = currentFrame.suffix; break; } else { final int cmp = term.compareTo(target); if (cmp < 0) { if (currentFrame.nextEnt == currentFrame.entCount) { currentFrame.startBytePos = saveStartBytePos; currentFrame.suffix = saveSuffix; currentFrame.suffixesReader.setPosition(savePos); currentFrame.termState.termBlockOrd = saveTermBlockOrd; currentFrame.isAutoPrefixTerm = saveIsAutoPrefixTerm;
public void scanToSubBlock(long subFP) { assert !isLeafBlock; //if (DEBUG) System.out.println(" scanToSubBlock fp=" + fp + " subFP=" + subFP + " entCount=" + entCount + " lastSubFP=" + lastSubFP); //assert nextEnt == 0; if (lastSubFP == subFP) { //if (DEBUG) System.out.println(" already positioned"); return; } assert subFP < fp : "fp=" + fp + " subFP=" + subFP; final long targetSubCode = fp - subFP; //if (DEBUG) System.out.println(" targetSubCode=" + targetSubCode); while(true) { assert nextEnt < entCount; nextEnt++; final int code = suffixesReader.readVInt(); suffixesReader.skipBytes(code >>> 1); if ((code & 1) != 0) { final long subCode = suffixesReader.readVLong(); if (targetSubCode == subCode) { //if (DEBUG) System.out.println(" match!"); lastSubFP = subFP; return; } } else { state.termBlockOrd++; } } }