postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); for (int pos = 0; pos < longsSize; pos++) { assert longs[pos] >= 0; postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); for (int pos = 0; pos < longsSize; pos++) { assert longs[pos] >= 0;
postingsWriter.init(termsOut, state); // have consumer write its format/header
TermsWriter(FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; assert fieldInfo.getIndexOptions() != IndexOptions.NONE; docsSeen = new FixedBitSet(maxDoc); this.longsSize = postingsWriter.setField(fieldInfo); this.longs = new long[longsSize]; }
/** Writes one term's worth of postings. */ public void write(BytesRef text, TermsEnum termsEnum) throws IOException { /* if (DEBUG) { int[] tmp = new int[lastTerm.length]; System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length); System.out.println("BTTW: write term=" + brToString(text) + " prefixStarts=" + Arrays.toString(tmp) + " pending.size()=" + pending.size()); } */ BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); if (state != null) { assert state.docFreq != 0; assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter; pushTerm(text); PendingTerm term = new PendingTerm(text, state); pending.add(term); //if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" + pending.size()); sumDocFreq += state.docFreq; sumTotalTermFreq += state.totalTermFreq; numTerms++; if (firstPendingTerm == null) { firstPendingTerm = term; } lastPendingTerm = term; } }
@Override public FieldsConsumer fieldsConsumer(final SegmentWriteState state) throws IOException { final PostingsWriterBase postingsWriter = new Siren10PostingsWriter(state, this.getFactory()); boolean success = false; try { final FieldsConsumer ret = new BlockTreeTermsWriter(state, postingsWriter, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); success = true; return ret; } finally { if (!success) { postingsWriter.close(); } } }
/** Writes one term's worth of postings. */ public void write(BytesRef text, TermsEnum termsEnum) throws IOException { BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); // TODO: LUCENE-5693: we don't need this check if we fix IW to not send deleted docs to us on flush: if (state != null && ((IDVersionPostingsWriter) postingsWriter).lastDocID != -1) { assert state.docFreq != 0; assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter; pushTerm(text); PendingTerm term = new PendingTerm(BytesRef.deepCopyOf(text), state); pending.add(term); numTerms++; if (firstPendingTerm == null) { firstPendingTerm = term; } lastPendingTerm = term; } }
@Override public FieldsConsumer fieldsConsumer(final SegmentWriteState state) throws IOException { final PostingsWriterBase docs = new Siren10PostingsWriter(state, this.getFactory()); boolean success = false; try { final FieldsConsumer ret = new BlockTreeTermsWriter(state, docs, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); success = true; return ret; } finally { if (!success) { docs.close(); } } }
TermsWriter(FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; docsSeen = new FixedBitSet(maxDoc); this.longsSize = postingsWriter.setField(fieldInfo); this.longs = new long[longsSize]; }
/** Writes one term's worth of postings. */ public void write(BytesRef text, TermsEnum termsEnum) throws IOException { /* if (DEBUG) { int[] tmp = new int[lastTerm.length]; System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length); System.out.println("BTTW: write term=" + brToString(text) + " prefixStarts=" + Arrays.toString(tmp) + " pending.size()=" + pending.size()); } */ BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); if (state != null) { assert state.docFreq != 0; assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter; pushTerm(text); PendingTerm term = new PendingTerm(text, state); pending.add(term); //if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" + pending.size()); sumDocFreq += state.docFreq; sumTotalTermFreq += state.totalTermFreq; numTerms++; if (firstPendingTerm == null) { firstPendingTerm = term; } lastPendingTerm = term; } }
public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION); this.postingsWriter = postingsWriter; this.fieldInfos = state.fieldInfos; this.out = state.directory.createOutput(termsFileName, state.context); this.maxDoc = state.segmentInfo.maxDoc(); boolean success = false; try { CodecUtil.writeIndexHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); this.postingsWriter.init(out, state); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); } } }
public void finishTerm(BytesRef text, BlockTermState state) throws IOException { // write term meta data into fst final FSTTermOutputs.TermData meta = new FSTTermOutputs.TermData(); meta.longs = new long[longsSize]; meta.bytes = null; meta.docFreq = state.docFreq; meta.totalTermFreq = state.totalTermFreq; postingsWriter.encodeTerm(meta.longs, metaWriter, fieldInfo, state, true); final int bytesSize = (int)metaWriter.getFilePointer(); if (bytesSize > 0) { meta.bytes = new byte[bytesSize]; metaWriter.writeTo(meta.bytes, 0); metaWriter.reset(); } builder.add(Util.toIntsRef(text, scratchTerm), meta); numTerms++; }
TermsWriter(FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; docsSeen = new FixedBitSet(maxDoc); this.longsSize = postingsWriter.setField(fieldInfo); this.longs = new long[longsSize]; }
/** Writes one term's worth of postings. */ public void write(BytesRef text, TermsEnum termsEnum, PrefixTerm prefixTerm) throws IOException { /* if (DEBUG) { int[] tmp = new int[lastTerm.length]; System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length); System.out.println("BTTW: write term=" + brToString(text) + " prefixStarts=" + Arrays.toString(tmp) + " pending.size()=" + pending.size()); } */ BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); if (state != null) { assert state.docFreq != 0; assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter; pushTerm(text); PendingTerm term = new PendingTerm(text, state, prefixTerm); pending.add(term); //if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" + pending.size()); if (prefixTerm == null) { // Only increment stats for real terms: sumDocFreq += state.docFreq; sumTotalTermFreq += state.totalTermFreq; numTerms++; if (firstPendingTerm == null) { firstPendingTerm = term; } lastPendingTerm = term; } } }
public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter, SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; maxDoc = state.segmentInfo.maxDoc(); out = state.directory.createOutput(termsFileName, state.context); boolean success = false; try { fieldInfos = state.fieldInfos; CodecUtil.writeIndexHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); currentField = null; this.postingsWriter = postingsWriter; // segment = state.segmentName; //System.out.println("BTW.init seg=" + state.segmentName); postingsWriter.init(out, state); // have consumer write its format/header success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); } } }
postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); for (int pos = 0; pos < longsSize; pos++) { assert longs[pos] >= 0; postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); for (int pos = 0; pos < longsSize; pos++) { assert longs[pos] >= 0;
TermsWriter(FieldInfo fieldInfo) { this.fieldInfo = fieldInfo; docsSeen = new FixedBitSet(maxDoc); this.longsSize = postingsWriter.setField(fieldInfo); this.longs = new long[longsSize]; }
/** Writes one term's worth of postings. */ public void write(BytesRef text, TermsEnum termsEnum, PrefixTerm prefixTerm) throws IOException { /* if (DEBUG) { int[] tmp = new int[lastTerm.length]; System.arraycopy(prefixStarts, 0, tmp, 0, tmp.length); System.out.println("BTTW: write term=" + brToString(text) + " prefixStarts=" + Arrays.toString(tmp) + " pending.size()=" + pending.size()); } */ BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); if (state != null) { assert state.docFreq != 0; assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter; pushTerm(text); PendingTerm term = new PendingTerm(text, state, prefixTerm); pending.add(term); //if (DEBUG) System.out.println(" add pending term = " + text + " pending.size()=" + pending.size()); if (prefixTerm == null) { // Only increment stats for real terms: sumDocFreq += state.docFreq; sumTotalTermFreq += state.totalTermFreq; numTerms++; if (firstPendingTerm == null) { firstPendingTerm = term; } lastPendingTerm = term; } } }
public FSTOrdTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION); final String termsBlockFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_BLOCK_EXTENSION); this.postingsWriter = postingsWriter; this.fieldInfos = state.fieldInfos; this.maxDoc = state.segmentInfo.maxDoc(); boolean success = false; try { this.indexOut = state.directory.createOutput(termsIndexFileName, state.context); this.blockOut = state.directory.createOutput(termsBlockFileName, state.context); CodecUtil.writeIndexHeader(indexOut, TERMS_INDEX_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); CodecUtil.writeIndexHeader(blockOut, TERMS_CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); this.postingsWriter.init(blockOut, state); success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(indexOut, blockOut); } } }
postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); for (int pos = 0; pos < longsSize; pos++) { assert longs[pos] >= 0; postingsWriter.encodeTerm(longs, bytesWriter, fieldInfo, state, absolute); for (int pos = 0; pos < longsSize; pos++) { assert longs[pos] >= 0;
TermsWriter(FieldInfo fieldInfo) { this.numTerms = 0; this.fieldInfo = fieldInfo; this.longsSize = postingsWriter.setField(fieldInfo); this.outputs = new FSTTermOutputs(fieldInfo, longsSize); this.builder = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs); }