/** add a term. This fully consumes in the incoming {@link BytesRef}. */ public void add(String field, BytesRef bytes) { assert lastTerm.equals(new Term("")) || new Term(field, bytes).compareTo(lastTerm) > 0; try { final int prefix; if (size > 0 && field.equals(lastTerm.field)) { // same field as the last term prefix = StringHelper.bytesDifference(lastTerm.bytes, bytes); output.writeVInt(prefix << 1); } else { // field change prefix = 0; output.writeVInt(1); output.writeString(field); } int suffix = bytes.length - prefix; output.writeVInt(suffix); output.writeBytes(bytes.bytes, bytes.offset + prefix, suffix); lastTermBytes.copyBytes(bytes); lastTerm.bytes = lastTermBytes.get(); lastTerm.field = field; size += 1; } catch (IOException e) { throw new RuntimeException(e); } }
scratchBytes.writeVInt(blocks.size()-1); for (int i=1;i<blocks.size();i++) { PendingBlock sub = blocks.get(i);
suffixWriter.writeVInt(suffix); suffixWriter.writeBytes(term.termBytes, prefixLength, suffix); assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >= floorLeadLabel; statsWriter.writeVInt(state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS) { assert state.totalTermFreq >= state.docFreq: state.totalTermFreq + " vs " + state.docFreq; suffixWriter.writeVInt(suffix << 1); suffixWriter.writeBytes(term.termBytes, prefixLength, suffix); statsWriter.writeVInt(state.docFreq); if (fieldInfo.getIndexOptions() != IndexOptions.DOCS) { assert state.totalTermFreq >= state.docFreq; suffixWriter.writeVInt((suffix<<1)|1); suffixWriter.writeBytes(block.prefix.bytes, prefixLength, suffix);
static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException { try (RAMOutputStream out = new RAMOutputStream()) { out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); if (searchShardTarget.getClusterAlias() != null) { out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId())); } else { out.writeString(searchShardTarget.getNodeId()); } } byte[] bytes = new byte[(int) out.getFilePointer()]; out.writeTo(bytes, 0); return Base64.getUrlEncoder().encodeToString(bytes); } }
public void startDoc(int docID, int termDocFreq) throws IOException { //System.out.println(" startDoc docID=" + docID + " freq=" + termDocFreq); final int delta = docID - lastDocID; assert docID == 0 || delta > 0; lastDocID = docID; docCount++; if (field.getIndexOptions() == IndexOptions.DOCS) { buffer.writeVInt(delta); } else if (termDocFreq == 1) { buffer.writeVInt((delta<<1) | 1); } else { buffer.writeVInt(delta<<1); assert termDocFreq > 0; buffer.writeVInt(termDocFreq); } lastPos = 0; lastOffset = 0; }
private void bufferSkip(int doc) throws IOException { long freqPointer = freqOutput.getFilePointer(); long proxPointer = proxOutput.getFilePointer(); skipBuffer.writeVInt(doc - lastSkipDoc); skipBuffer.writeVInt((int) (freqPointer - lastSkipFreqPointer)); skipBuffer.writeVInt((int) (proxPointer - lastSkipProxPointer)); lastSkipDoc = doc; lastSkipFreqPointer = freqPointer; lastSkipProxPointer = proxPointer; }
if (payloadLen != lastPayloadLen) { lastPayloadLen = payloadLen; buffer.writeVInt((delta<<1)|1); buffer.writeVInt(payloadLen); } else { buffer.writeVInt(delta<<1); buffer.writeVInt(delta); int offsetLength = endOffset - startOffset; if (offsetLength != lastOffsetLength) { buffer.writeVInt(offsetDelta << 1 | 1); buffer.writeVInt(offsetLength); } else { buffer.writeVInt(offsetDelta << 1);
/** add a term */ public void add(String field, BytesRef bytes) { assert lastTerm.equals(new Term("")) || new Term(field, bytes).compareTo(lastTerm) > 0; try { int prefix = sharedPrefix(lastTerm.bytes, bytes); int suffix = bytes.length - prefix; if (field.equals(lastTerm.field)) { output.writeVInt(prefix << 1); } else { output.writeVInt(prefix << 1 | 1); output.writeString(field); } output.writeVInt(suffix); output.writeBytes(bytes.bytes, bytes.offset + prefix, suffix); lastTermBytes.copyBytes(bytes); lastTerm.bytes = lastTermBytes.get(); lastTerm.field = field; size += 1; } catch (IOException e) { throw new RuntimeException(e); } }
/** add a term */ public void add(String field, BytesRef bytes) { assert lastTerm.equals(new Term("")) || new Term(field, bytes).compareTo(lastTerm) > 0; try { int prefix = sharedPrefix(lastTerm.bytes, bytes); int suffix = bytes.length - prefix; if (field.equals(lastTerm.field)) { output.writeVInt(prefix << 1); } else { output.writeVInt(prefix << 1 | 1); output.writeString(field); } output.writeVInt(suffix); output.writeBytes(bytes.bytes, bytes.offset + prefix, suffix); lastTermBytes.copyBytes(bytes); lastTerm.bytes = lastTermBytes.get(); lastTerm.field = field; size += 1; } catch (IOException e) { throw new RuntimeException(e); } }
/** add a term. This fully consumes in the incoming {@link BytesRef}. */ public void add(String field, BytesRef bytes) { assert lastTerm.equals(new Term("")) || new Term(field, bytes).compareTo(lastTerm) > 0; try { final int prefix; if (size > 0 && field.equals(lastTerm.field)) { // same field as the last term prefix = StringHelper.bytesDifference(lastTerm.bytes, bytes); output.writeVInt(prefix << 1); } else { // field change prefix = 0; output.writeVInt(1); output.writeString(field); } int suffix = bytes.length - prefix; output.writeVInt(suffix); output.writeBytes(bytes.bytes, bytes.offset + prefix, suffix); lastTermBytes.copyBytes(bytes); lastTerm.bytes = lastTermBytes.get(); lastTerm.field = field; size += 1; } catch (IOException e) { throw new RuntimeException(e); } }
public void finishTerm(BytesRef text, BlockTermState state) throws IOException { if (numTerms > 0 && numTerms % SKIP_INTERVAL == 0) { bufferSkip(); } // write term meta data into fst final long longs[] = new long[longsSize]; final long delta = state.totalTermFreq - state.docFreq; if (state.totalTermFreq > 0) { if (delta == 0) { statsOut.writeVInt(state.docFreq<<1|1); } else { statsOut.writeVInt(state.docFreq<<1); statsOut.writeVLong(state.totalTermFreq-state.docFreq); } } else { statsOut.writeVInt(state.docFreq); } postingsWriter.encodeTerm(longs, metaBytesOut, fieldInfo, state, true); for (int i = 0; i < longsSize; i++) { metaLongsOut.writeVLong(longs[i] - lastLongs[i]); lastLongs[i] = longs[i]; } metaLongsOut.writeVLong(metaBytesOut.getFilePointer() - lastMetaBytesFP); builder.add(Util.toIntsRef(text, scratchTerm), numTerms); numTerms++; lastMetaBytesFP = metaBytesOut.getFilePointer(); }
indexBytesWriter.writeVInt(t.blockCount);
static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException { try (RAMOutputStream out = new RAMOutputStream()) { out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); out.writeString(searchPhaseResult.getSearchShardTarget().getNodeId()); } byte[] bytes = new byte[(int) out.getFilePointer()]; out.writeTo(bytes, 0); return Base64.getUrlEncoder().encodeToString(bytes); } }
static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException { try (RAMOutputStream out = new RAMOutputStream()) { out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); if (searchShardTarget.getClusterAlias() != null) { out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId())); } else { out.writeString(searchShardTarget.getNodeId()); } } byte[] bytes = new byte[(int) out.getFilePointer()]; out.writeTo(bytes, 0); return Base64.getUrlEncoder().encodeToString(bytes); } }
static String buildScrollId(AtomicArray<? extends SearchPhaseResult> searchPhaseResults) throws IOException { try (RAMOutputStream out = new RAMOutputStream()) { out.writeString(searchPhaseResults.length() == 1 ? ParsedScrollId.QUERY_AND_FETCH_TYPE : ParsedScrollId.QUERY_THEN_FETCH_TYPE); out.writeVInt(searchPhaseResults.asList().size()); for (SearchPhaseResult searchPhaseResult : searchPhaseResults.asList()) { out.writeLong(searchPhaseResult.getRequestId()); SearchShardTarget searchShardTarget = searchPhaseResult.getSearchShardTarget(); if (searchShardTarget.getClusterAlias() != null) { out.writeString(RemoteClusterAware.buildRemoteIndexName(searchShardTarget.getClusterAlias(), searchShardTarget.getNodeId())); } else { out.writeString(searchShardTarget.getNodeId()); } } byte[] bytes = new byte[(int) out.getFilePointer()]; out.writeTo(bytes, 0); return Base64.getUrlEncoder().encodeToString(bytes); } }
private void finishTerm(BytesRef text, TermStats stats) throws IOException { if (stats.docFreq == 0) { return; } assert postingsWriter.docCount == stats.docFreq; assert buffer2.getFilePointer() == 0; buffer2.writeVInt(stats.docFreq); if (field.getIndexOptions() != IndexOptions.DOCS) { buffer2.writeVLong(stats.totalTermFreq-stats.docFreq); } int pos = (int) buffer2.getFilePointer(); buffer2.writeTo(finalBuffer, 0); buffer2.reset(); final int totalBytes = pos + (int) postingsWriter.buffer.getFilePointer(); if (totalBytes > finalBuffer.length) { finalBuffer = ArrayUtil.grow(finalBuffer, totalBytes); } postingsWriter.buffer.writeTo(finalBuffer, pos); postingsWriter.buffer.reset(); spare.bytes = finalBuffer; spare.length = totalBytes; //System.out.println(" finishTerm term=" + text.utf8ToString() + " " + totalBytes + " bytes totalTF=" + stats.totalTermFreq); //for(int i=0;i<totalBytes;i++) { // System.out.println(" " + Integer.toHexString(finalBuffer[i]&0xFF)); //} builder.add(Util.toIntsRef(text, scratchIntsRef), BytesRef.deepCopyOf(spare)); termCount++; }