/** Writes the terms file trailer. */ private void writeTrailer(IndexOutput out, long dirStart) throws IOException { out.writeLong(dirStart); }
/** Writes the index file trailer. */ private void writeIndexTrailer(IndexOutput indexOut, long dirStart) throws IOException { indexOut.writeLong(dirStart); }
/** * Writes CRC32 value as a 64-bit long to the output. * @throws IllegalStateException if CRC is formatted incorrectly (wrong bits set) * @throws IOException if an i/o error occurs */ static void writeCRC(IndexOutput output) throws IOException { long value = output.getChecksum(); if ((value & 0xFFFFFFFF00000000L) != 0) { throw new IllegalStateException("Illegal CRC-32 checksum: " + value + " (resource=" + output + ")"); } output.writeLong(value); } }
meta.writeLong(-2); meta.writeLong(0L); } else if (numDocsWithValue == maxDoc) { meta.writeLong(-1); meta.writeLong(0L); } else { long offset = data.getFilePointer(); meta.writeLong(offset); values = normsProducer.getNorms(field); IndexedDISI.writeBitSet(values, data); meta.writeLong(data.getFilePointer() - offset); meta.writeLong(min); } else { meta.writeLong(data.getFilePointer()); values = normsProducer.getNorms(field); writeValues(values, numBytesPerValue, data);
@Override public void append(byte[] packedValue, long ord, int docID) throws IOException { assert packedValue.length == packedBytesLength; out.writeBytes(packedValue, 0, packedValue.length); out.writeInt(docID); if (singleValuePerDoc == false) { if (longOrds) { out.writeLong(ord); } else { assert ord <= Integer.MAX_VALUE; out.writeInt((int) ord); } } count++; assert expectedCount == 0 || count <= expectedCount; }
meta.writeLong(-2); meta.writeLong(0L); } else if (numDocsWithField == maxDoc) { meta.writeLong(-1); meta.writeLong(0L); } else { long offset = data.getFilePointer(); meta.writeLong(offset); values = valuesProducer.getSorted(field); IndexedDISI.writeBitSet(values, data); meta.writeLong(data.getFilePointer() - offset); if (values.getValueCount() <= 1) { meta.writeByte((byte) 0); meta.writeLong(0L); meta.writeLong(0L); } else { int numberOfBitsPerOrd = DirectWriter.unsignedBitsRequired(values.getValueCount() - 1); meta.writeByte((byte) numberOfBitsPerOrd); long start = data.getFilePointer(); meta.writeLong(start); DirectWriter writer = DirectWriter.getInstance(data, numDocsWithField, numberOfBitsPerOrd); values = valuesProducer.getSorted(field); meta.writeLong(data.getFilePointer() - start);
meta.writeLong(start); int numDocsWithField = 0; int minLength = Integer.MAX_VALUE; meta.writeLong(data.getFilePointer() - start); meta.writeLong(-2); meta.writeLong(0L); } else if (numDocsWithField == maxDoc) { meta.writeLong(-1); meta.writeLong(0L); } else { long offset = data.getFilePointer(); meta.writeLong(offset); values = valuesProducer.getBinary(field); IndexedDISI.writeBitSet(values, data); meta.writeLong(data.getFilePointer() - offset); if (maxLength > minLength) { start = data.getFilePointer(); meta.writeLong(start); meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); meta.writeLong(data.getFilePointer() - start);
private void writeValues(NumericDocValues values, int numBytesPerValue, IndexOutput out) throws IOException, AssertionError { for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { long value = values.longValue(); switch (numBytesPerValue) { case 1: out.writeByte((byte) value); break; case 2: out.writeShort((short) value); break; case 4: out.writeInt((int) value); break; case 8: out.writeLong(value); break; default: throw new AssertionError(); } } } }
private void writeBlock(long[] values, int length, long gcd, GrowableByteArrayDataOutput buffer) throws IOException { assert length > 0; long min = values[0]; long max = values[0]; for (int i = 1; i < length; ++i) { final long v = values[i]; assert Math.floorMod(values[i] - min, gcd) == 0; min = Math.min(min, v); max = Math.max(max, v); } if (min == max) { data.writeByte((byte) 0); data.writeLong(min); } else { final int bitsPerValue = DirectWriter.unsignedBitsRequired(max - min); buffer.reset(); assert buffer.getPosition() == 0; final DirectWriter w = DirectWriter.getInstance(buffer, length, bitsPerValue); for (int i = 0; i < length; ++i) { w.add((values[i] - min) / gcd); } w.finish(); data.writeByte((byte) bitsPerValue); data.writeLong(min); data.writeInt(buffer.getPosition()); data.writeBytes(buffer.getBytes(), buffer.getPosition()); } }
private static void flush(int block, FixedBitSet buffer, int cardinality, IndexOutput out) throws IOException { assert block >= 0 && block < 65536; out.writeShort((short) block); assert cardinality > 0 && cardinality <= 65536; out.writeShort((short) (cardinality - 1)); if (cardinality > MAX_ARRAY_LENGTH) { if (cardinality != 65536) { // all docs are set for (long word : buffer.getBits()) { out.writeLong(word); } } } else { BitSetIterator it = new BitSetIterator(buffer, cardinality); for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) { out.writeShort((short) doc); } } }
@Override public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED_NUMERIC); long[] stats = writeValues(field, valuesProducer); int numDocsWithField = Math.toIntExact(stats[0]); long numValues = stats[1]; assert numValues >= numDocsWithField; meta.writeInt(numDocsWithField); if (numValues > numDocsWithField) { long start = data.getFilePointer(); meta.writeLong(start); meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT); long addr = 0; addressesWriter.add(addr); SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { addr += values.docValueCount(); addressesWriter.add(addr); } addressesWriter.finish(); meta.writeLong(data.getFilePointer() - start); } }
@Override public void writeLiveDocs(Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException { long gen = info.getNextDelGen(); String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen); int delCount = 0; try (IndexOutput output = dir.createOutput(name, context)) { CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), Long.toString(gen, Character.MAX_RADIX)); final int longCount = FixedBitSet.bits2words(bits.length()); for (int i = 0; i < longCount; ++i) { long currentBits = 0; for (int j = i << 6, end = Math.min(j + 63, bits.length() - 1); j <= end; ++j) { if (bits.get(j)) { currentBits |= 1L << j; // mod 64 } else { delCount += 1; } } output.writeLong(currentBits); } CodecUtil.writeFooter(output); } if (delCount != info.getDelCount() + newDelCount) { throw new CorruptIndexException("bits.deleted=" + delCount + " info.delcount=" + info.getDelCount() + " newdelcount=" + newDelCount, name); } }
meta.writeLong(-1); meta.writeLong(0L); } else { long offset = data.getFilePointer(); meta.writeLong(offset); values = valuesProducer.getSortedSet(field); IndexedDISI.writeBitSet(values, data); meta.writeLong(data.getFilePointer() - offset); meta.writeByte((byte) numberOfBitsPerOrd); long start = data.getFilePointer(); meta.writeLong(start); DirectWriter writer = DirectWriter.getInstance(data, numOrds, numberOfBitsPerOrd); values = valuesProducer.getSortedSet(field); meta.writeLong(data.getFilePointer() - start); meta.writeLong(start); meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); meta.writeLong(data.getFilePointer() - start);
out.writeLong(version); out.writeLong(siPerCommit.getDelGen()); int delCount = siPerCommit.getDelCount(); if (delCount < 0 || delCount > si.maxDoc()) { out.writeLong(siPerCommit.getFieldInfosGen()); out.writeLong(siPerCommit.getDocValuesGen()); int softDelCount = siPerCommit.getSoftDelCount(); if (softDelCount < 0 || softDelCount > si.maxDoc()) {
@Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size()); for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeString(fi.name); output.writeVInt(fi.number); byte bits = 0x0; if (fi.hasVectors()) bits |= STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= OMIT_NORMS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); } CodecUtil.writeFooter(output); } }
output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); output.writeVInt(fi.getPointDataDimensionCount());