/** Add a new element to this builder. */ public Builder add(long l) { if (pending == null) { throw new IllegalStateException("Cannot be reused after build()"); } if (pendingOff == pending.length) { // check size if (values.length == valuesOff) { final int newLength = ArrayUtil.oversize(valuesOff + 1, 8); grow(newLength); } pack(); } pending[pendingOff++] = l; size += 1; return this; }
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) { final long termOrd = builder.nextOrdinal(); assert termOrd == termOrdToBytesOffset.size(); termOrdToBytesOffset.add(bytes.copyUsingLengthPrefix(term)); docsEnum = termsEnum.postings(docsEnum, PostingsEnum.NONE); for (int docId = docsEnum.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docsEnum.nextDoc()) { final Ordinals ordinals = builder.build(); data = new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffset.build(), ordinals); success = true; return data;
for (BytesRef term = termsEnum.next(); term != null; term = termsEnum.next()) { final long termOrd = builder.nextOrdinal(); assert termOrd == termOrdToBytesOffset.size(); termOrdToBytesOffset.add(bytes.copyUsingLengthPrefix(term)); docsEnum = termsEnum.postings(docsEnum, PostingsEnum.NONE); for (int docId = docsEnum.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docsEnum.nextDoc()) { final Ordinals ordinals = builder.build(); data = new PagedBytesAtomicFieldData(bytesReader, termOrdToBytesOffset.build(), ordinals); success = true; return data;
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pending.size() == maxDoc; final int valueCount = hash.size(); final PackedLongValues ords = pending.build(); final int[] sortedValues = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator()); final int[] ordMap = new int[valueCount]; for(int ord=0;ord<valueCount;ord++) { ordMap[sortedValues[ord]] = ord; } dvConsumer.addSortedField(fieldInfo, // ord -> value new Iterable<BytesRef>() { @Override public Iterator<BytesRef> iterator() { return new ValuesIterator(sortedValues, valueCount, hash); } }, // doc -> ord new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new OrdsIterator(ordMap, maxDoc, ords); } }); }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pending.size() == maxDoc; final int valueCount = hash.size(); final PackedLongValues ords = pending.build(); final int[] sortedValues = hash.sort(BytesRef.getUTF8SortedAsUnicodeComparator()); final int[] ordMap = new int[valueCount]; for(int ord=0;ord<valueCount;ord++) { ordMap[sortedValues[ord]] = ord; } dvConsumer.addSortedField(fieldInfo, // ord -> value new Iterable<BytesRef>() { @Override public Iterator<BytesRef> iterator() { return new ValuesIterator(sortedValues, valueCount, hash); } }, // doc -> ord new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new OrdsIterator(ordMap, maxDoc, ords); } }); }
public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pendingCounts.size() == maxDoc; final PackedLongValues values = pending.build(); final PackedLongValues valueCounts = pendingCounts.build(); dvConsumer.addSortedNumericField(fieldInfo, // doc -> valueCount new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new CountIterator(valueCounts); } }, // values new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new ValuesIterator(values); } }); }
public void mergeBuckets(long[] mergeMap) { List<Entry> newEntries = new ArrayList<>(entries.size()); for (Entry sourceEntry : entries) { PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = sourceEntry.buckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } newEntries.add(new Entry(sourceEntry.context, sourceEntry.docDeltas, newBuckets.build())); } entries = newEntries; // if there are buckets that have been collected in the current segment // we need to update the bucket ordinals there too if (buckets.size() > 0) { PackedLongValues currentBuckets = buckets.build(); PackedLongValues.Builder newBuckets = PackedLongValues.packedBuilder(PackedInts.DEFAULT); for (PackedLongValues.Iterator itr = currentBuckets.iterator(); itr.hasNext();) { long bucket = itr.next(); newBuckets.add(mergeMap[Math.toIntExact(bucket)]); } buckets = newBuckets; } }
@Override public void flush(SegmentWriteState state, DocValuesConsumer dvConsumer) throws IOException { final int maxDoc = state.segmentInfo.maxDoc(); assert pendingCounts.size() == maxDoc; final PackedLongValues values = pending.build(); final PackedLongValues valueCounts = pendingCounts.build(); dvConsumer.addSortedNumericField(fieldInfo, // doc -> valueCount new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new CountIterator(valueCounts); } }, // values new Iterable<Number>() { @Override public Iterator<Number> iterator() { return new ValuesIterator(values); } }); }
/** Add a new element to this builder. */ public Builder add(long l) { if (pending == null) { throw new IllegalStateException("Cannot be reused after build()"); } if (pendingOff == pending.length) { // check size if (values.length == valuesOff) { final int newLength = ArrayUtil.oversize(valuesOff + 1, 8); grow(newLength); } pack(); } pending[pendingOff++] = l; size += 1; return this; }
/** Add a new element to this builder. */ public Builder add(long l) { if (pending == null) { throw new IllegalStateException("Cannot be reused after build()"); } if (pendingOff == pending.length) { // check size if (values.length == valuesOff) { final int newLength = ArrayUtil.oversize(valuesOff + 1, 8); grow(newLength); } pack(); } pending[pendingOff++] = l; size += 1; return this; }
/** Add a new element to this builder. */ public Builder add(long l) { if (pending == null) { throw new IllegalStateException("Cannot be reused after build()"); } if (pendingOff == pending.length) { // check size if (values.length == valuesOff) { final int newLength = ArrayUtil.oversize(valuesOff + 1, 8); grow(newLength); } pack(); } pending[pendingOff++] = l; size += 1; return this; }
public void addValue(int docID, BytesRef value) { if (docID < pending.size()) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } if (value == null) { throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": null value not allowed"); } if (value.length > (BYTE_BLOCK_SIZE - 2)) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" is too large, must be <= " + (BYTE_BLOCK_SIZE - 2)); } // Fill in any holes: while(pending.size() < docID) { pending.add(EMPTY_ORD); } addOneValue(value); }
public void addValue(int docID, BytesRef value) { if (docID < pending.size()) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" appears more than once in this document (only one value is allowed per field)"); } if (value == null) { throw new IllegalArgumentException("field \"" + fieldInfo.name + "\": null value not allowed"); } if (value.length > (BYTE_BLOCK_SIZE - 2)) { throw new IllegalArgumentException("DocValuesField \"" + fieldInfo.name + "\" is too large, must be <= " + (BYTE_BLOCK_SIZE - 2)); } // Fill in any holes: while(pending.size() < docID) { pending.add(EMPTY_ORD); } addOneValue(value); }
/** Return a new {@link Builder} that will compress efficiently positive integers. */ public static PackedLongValues.Builder packedBuilder(int pageSize, float acceptableOverheadRatio) { return new PackedLongValues.Builder(pageSize, acceptableOverheadRatio); }
Builder(int pageSize, float acceptableOverheadRatio) { pageShift = checkBlockSize(pageSize, MIN_PAGE_SIZE, MAX_PAGE_SIZE); pageMask = pageSize - 1; this.acceptableOverheadRatio = acceptableOverheadRatio; values = new PackedInts.Reader[INITIAL_PAGE_COUNT]; pending = new long[pageSize]; valuesOff = 0; pendingOff = 0; size = 0; ramBytesUsed = baseRamBytesUsed() + RamUsageEstimator.sizeOf(pending) + RamUsageEstimator.shallowSizeOf(values); }