public void addValue(int docID, BytesRef value) { if (docID <= lastDocID) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } if (value == null) { throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\": null value not allowed"); } if (value.length > MAX_LENGTH) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" is too large, must be <= " + MAX_LENGTH); } maxLength = Math.max(value.length, maxLength); lengths.add(value.length); try { bytesOut.writeBytes(value.bytes, value.offset, value.length); } catch (IOException ioe) { // Should never happen! throw new RuntimeException(ioe); } docsWithField.add(docID); updateBytesUsed(); lastDocID = docID; }
private void finishCurrentDoc() { if (currentDoc == -1) { return; } Arrays.sort(currentValues, 0, currentUpto); int lastValue = -1; int count = 0; for (int i = 0; i < currentUpto; i++) { int termID = currentValues[i]; // if it's not a duplicate if (termID != lastValue) { pending.add(termID); // record the term id count++; } lastValue = termID; } // record the number of unique term ids for this doc pendingCounts.add(count); maxCount = Math.max(maxCount, count); currentUpto = 0; docsWithField.add(currentDoc); }
public void addValue(int docID, long value) { if (docID <= lastDocID) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } pending.add(value); docsWithField.add(docID); updateBytesUsed(); lastDocID = docID; }
@Override public void finish(int maxDoc) { while(pending.size() < maxDoc) { pending.add(EMPTY_ORD); } updateBytesUsed(); }
public void addValue(int docID, long value) { if (docID != currentDoc) { finishCurrentDoc(); } // Fill in any holes: while(currentDoc < docID) { pendingCounts.add(0); // no values currentDoc++; } addOneValue(value); updateBytesUsed(); }
final long termOrd = builder.nextOrdinal(); assert termOrd == termOrdToBytesOffset.size(); termOrdToBytesOffset.add(bytes.copyUsingLengthPrefix(term)); docsEnum = termsEnum.postings(docsEnum, PostingsEnum.NONE); for (int docId = docsEnum.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docsEnum.nextDoc()) {
public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
public MultiOrdinals(OrdinalsBuilder builder, float acceptableOverheadRatio) { multiValued = builder.getNumMultiValuesDocs() > 0; valueCount = builder.getValueCount(); PackedLongValues.Builder endOffsetsBuilder = PackedLongValues.monotonicBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); PackedLongValues.Builder ordsBuilder = PackedLongValues.packedBuilder(OFFSETS_PAGE_SIZE, acceptableOverheadRatio); long lastEndOffset = 0; for (int i = 0; i < builder.maxDoc(); ++i) { final LongsRef docOrds = builder.docOrds(i); final long endOffset = lastEndOffset + docOrds.length; endOffsetsBuilder.add(endOffset); for (int j = 0; j < docOrds.length; ++j) { ordsBuilder.add(docOrds.longs[docOrds.offset + j]); } lastEndOffset = endOffset; } endOffsets = endOffsetsBuilder.build(); ords = ordsBuilder.build(); assert endOffsets.size() == builder.maxDoc(); assert ords.size() == builder.getTotalNumOrds() : ords.size() + " != " + builder.getTotalNumOrds(); }
builders[top.readerIndex].add(mappedDocID); if (top.liveDocs == null || top.liveDocs.get(top.docID)) { mappedDocID++;
private void addOneValue(BytesRef value) { int termID = hash.add(value); if (termID < 0) { termID = -termID-1; } else { // reserve additional space for each unique value: // 1. when indexing, when hash is 50% full, rehash() suddenly needs 2*size ints. // TODO: can this same OOM happen in THPF? // 2. when flushing, we need 1 int per value (slot in the ordMap). iwBytesUsed.addAndGet(2 * Integer.BYTES); } pending.add(termID); updateBytesUsed(); }
private void finishCurrentDoc() { if (currentDoc == -1) { return; } Arrays.sort(currentValues, 0, currentUpto); for (int i = 0; i < currentUpto; i++) { pending.add(currentValues[i]); } // record the number of values for this doc pendingCounts.add(currentUpto); currentUpto = 0; docsWithField.add(currentDoc); }
public void addValue(int docID, long value) { if (docID <= lastDocID) { throw new IllegalArgumentException("Norm for \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } pending.add(value); docsWithField.add(docID); updateBytesUsed(); lastDocID = docID; }
static PackedLongValues removeDeletes(final int maxDoc, final Bits liveDocs) { final PackedLongValues.Builder docMapBuilder = PackedLongValues.monotonicBuilder(PackedInts.COMPACT); int del = 0; for (int i = 0; i < maxDoc; ++i) { docMapBuilder.add(i - del); if (liveDocs.get(i) == false) { ++del; } } return docMapBuilder.build(); } }