@Override public void writeByte(byte b) throws IOException { bytesSinceLastPause++; checkRate(); delegate.writeByte(b); }
/** * Write a block of data (<code>For</code> format). * * @param data the data to write * @param encoded a buffer to use to encode data * @param out the destination output * @throws IOException If there is a low-level I/O error */ void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { if (isAllEqual(data)) { out.writeByte((byte) ALL_VALUES_EQUAL); out.writeVInt(data[0]); return; } final int numBits = bitsRequired(data); assert numBits > 0 && numBits <= 32 : numBits; final PackedInts.Encoder encoder = encoders[numBits]; final int iters = iterations[numBits]; assert iters * encoder.byteValueCount() >= BLOCK_SIZE; final int encodedSize = encodedSizes[numBits]; assert iters * encoder.byteBlockCount() >= encodedSize; out.writeByte((byte) numBits); encoder.encode(data, 0, encoded, 0, iters); out.writeBytes(encoded, encodedSize); }
private void writeBlock(long[] values, int length, long gcd, GrowableByteArrayDataOutput buffer) throws IOException { assert length > 0; long min = values[0]; long max = values[0]; for (int i = 1; i < length; ++i) { final long v = values[i]; assert Math.floorMod(values[i] - min, gcd) == 0; min = Math.min(min, v); max = Math.max(max, v); } if (min == max) { data.writeByte((byte) 0); data.writeLong(min); } else { final int bitsPerValue = DirectWriter.unsignedBitsRequired(max - min); buffer.reset(); assert buffer.getPosition() == 0; final DirectWriter w = DirectWriter.getInstance(buffer, length, bitsPerValue); for (int i = 0; i < length; ++i) { w.add((values[i] - min) / gcd); } w.finish(); data.writeByte((byte) bitsPerValue); data.writeLong(min); data.writeInt(buffer.getPosition()); data.writeBytes(buffer.getBytes(), buffer.getPosition()); } }
@Override public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.NUMERIC); writeValues(field, new EmptyDocValuesProducer() { @Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { return DocValues.singleton(valuesProducer.getNumeric(field)); } }); }
assert suffixLength > 0; // terms are unique data.writeByte((byte) (Math.min(prefixLength, 15) | (Math.min(15, suffixLength - 1) << 4))); if (prefixLength >= 15) { data.writeVInt(prefixLength - 15);
@Override public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED); doAddSortedField(field, valuesProducer); }
@Override public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.BINARY);
/** Returns a sorted array containing unique field numbers */ private int[] flushFieldNums() throws IOException { SortedSet<Integer> fieldNums = new TreeSet<>(); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { fieldNums.add(fd.fieldNum); } } final int numDistinctFields = fieldNums.size(); assert numDistinctFields > 0; final int bitsRequired = PackedInts.bitsRequired(fieldNums.last()); final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired; vectorsStream.writeByte((byte) token); if (numDistinctFields - 1 >= 0x07) { vectorsStream.writeVInt(numDistinctFields - 1 - 0x07); } final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1); for (Integer fieldNum : fieldNums) { writer.add(fieldNum); } writer.finish(); int[] fns = new int[fieldNums.size()]; int i = 0; for (Integer key : fieldNums) { fns[i++] = key; } return fns; }
private void writeValues(NumericDocValues values, int numBytesPerValue, IndexOutput out) throws IOException, AssertionError { for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { long value = values.longValue(); switch (numBytesPerValue) { case 1: out.writeByte((byte) value); break; case 2: out.writeShort((short) value); break; case 4: out.writeInt((int) value); break; case 8: out.writeLong(value); break; default: throw new AssertionError(); } } } }
output.writeByte((byte) 1); Version minVersion = si.getMinVersion(); output.writeInt(minVersion.major); output.writeInt(minVersion.bugfix); } else { output.writeByte((byte) 0); output.writeByte((byte) (si.getUseCompoundFile() ? SegmentInfo.YES : SegmentInfo.NO)); output.writeMapOfStrings(si.getDiagnostics()); Set<String> files = si.files(); SortedSetSortField ssf = (SortedSetSortField) sortField; if (ssf.getSelector() == SortedSetSelector.Type.MIN) { output.writeByte((byte) 0); } else if (ssf.getSelector() == SortedSetSelector.Type.MAX) { output.writeByte((byte) 1); } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MIN) { output.writeByte((byte) 2); } else if (ssf.getSelector() == SortedSetSelector.Type.MIDDLE_MAX) { output.writeByte((byte) 3); } else { throw new IllegalStateException("Unexpected SortedSetSelector type: " + ssf.getSelector()); SortedNumericSortField snsf = (SortedNumericSortField) sortField; if (snsf.getNumericType() == SortField.Type.LONG) { output.writeByte((byte) 0); } else if (snsf.getNumericType() == SortField.Type.INT) { output.writeByte((byte) 1);
@Override public void writeByte(byte b) throws IOException { out.writeByte(b); }
@Override public void write(int b) throws IOException { out.writeByte((byte) b); }
@Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size()); for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeString(fi.name); output.writeVInt(fi.number); byte bits = 0x0; if (fi.hasVectors()) bits |= STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= OMIT_NORMS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); } CodecUtil.writeFooter(output); } }
if (fi.hasPayloads()) bits |= STORE_PAYLOADS; if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes());
@Override public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED_NUMERIC); long[] stats = writeValues(field, valuesProducer); int numDocsWithField = Math.toIntExact(stats[0]); long numValues = stats[1]; assert numValues >= numDocsWithField; meta.writeInt(numDocsWithField); if (numValues > numDocsWithField) { long start = data.getFilePointer(); meta.writeLong(start); meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT); long addr = 0; addressesWriter.add(addr); SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { addr += values.docValueCount(); addressesWriter.add(addr); } addressesWriter.finish(); meta.writeLong(data.getFilePointer() - start); } }
meta.writeByte((byte) numBitsPerValue); meta.writeLong(min); meta.writeLong(gcd);
int numBytesPerValue = numBytesPerValue(min, max); meta.writeByte((byte) numBytesPerValue); if (numBytesPerValue == 0) { meta.writeLong(min);
@Override public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED_SET); meta.writeByte((byte) 0); doAddSortedField(field, new EmptyDocValuesProducer() { @Override return; meta.writeByte((byte) 1); meta.writeByte((byte) numberOfBitsPerOrd); long start = data.getFilePointer(); meta.writeLong(start);