private static void writeBytesRef(IndexOutput out, BytesRef bytes) throws IOException { out.writeVInt(bytes.length); out.writeBytes(bytes.bytes, bytes.offset, bytes.length); } }
private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex) throws IOException { CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(numDataDims); out.writeVInt(numIndexDims); out.writeVInt(countPerLeaf); out.writeVInt(bytesPerDim); assert numLeaves > 0; out.writeVInt(numLeaves); out.writeBytes(minPackedValue, 0, packedIndexBytesLength); out.writeBytes(maxPackedValue, 0, packedIndexBytesLength); out.writeVLong(pointCount); out.writeVInt(docsSeen.cardinality()); out.writeVInt(packedIndex.length); out.writeBytes(packedIndex, 0, packedIndex.length); }
private void writeHeader(int docBase, int numBufferedDocs, int[] numStoredFields, int[] lengths, boolean sliced) throws IOException { final int slicedBit = sliced ? 1 : 0; // save docBase and numBufferedDocs fieldsStream.writeVInt(docBase); fieldsStream.writeVInt((numBufferedDocs) << 1 | slicedBit); // save numStoredFields saveInts(numStoredFields, numBufferedDocs, fieldsStream); // save lengths saveInts(lengths, numBufferedDocs, fieldsStream); }
CompressingStoredFieldsIndexWriter(IndexOutput indexOutput, int blockSize) throws IOException { if (blockSize <= 0) { throw new IllegalArgumentException("blockSize must be positive"); } this.blockSize = blockSize; this.fieldsIndexOut = indexOutput; reset(); totalDocs = 0; docBaseDeltas = new int[blockSize]; startPointerDeltas = new long[blockSize]; fieldsIndexOut.writeVInt(PackedInts.VERSION_CURRENT); }
@Override protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { int delta = curDoc - lastSkipDoc[level]; skipBuffer.writeVInt(delta); lastSkipDoc[level] = curDoc; skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]); lastSkipDocPointer[level] = curDocPointer; if (fieldHasPositions) { skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]); lastSkipPosPointer[level] = curPosPointer; skipBuffer.writeVInt(curPosBufferUpto); if (fieldHasPayloads) { skipBuffer.writeVInt(curPayloadByteUpto); } if (fieldHasOffsets || fieldHasPayloads) { skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]); lastSkipPayPointer[level] = curPayPointer; } } } }
private void addPositions(final PostingsEnum in, final IndexOutput out) throws IOException { int freq = in.freq(); out.writeVInt(freq); int previousPosition = 0; int previousEndOffset = 0; for (int i = 0; i < freq; i++) { final int pos = in.nextPosition(); final BytesRef payload = in.getPayload(); // The low-order bit of token is set only if there is a payload, the // previous bits are the delta-encoded position. final int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1); out.writeVInt(token); previousPosition = pos; if (storeOffsets) { // don't encode offsets if they are not stored final int startOffset = in.startOffset(); final int endOffset = in.endOffset(); out.writeVInt(startOffset - previousEndOffset); out.writeVInt(endOffset - startOffset); previousEndOffset = endOffset; } if (payload != null) { out.writeVInt(payload.length); out.writeBytes(payload.bytes, payload.offset, payload.length); } } }
@Override public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException { CodecUtil.writeIndexHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); termsOut.writeVInt(BLOCK_SIZE); }
void finish(int numDocs, long maxPointer) throws IOException { if (numDocs != totalDocs) { throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs); } if (blockChunks > 0) { writeBlock(); } fieldsIndexOut.writeVInt(0); // end marker fieldsIndexOut.writeVLong(maxPointer); CodecUtil.writeFooter(fieldsIndexOut); }
private int flushNumFields(int chunkDocs) throws IOException { if (chunkDocs == 1) { final int numFields = pendingDocs.getFirst().numFields; vectorsStream.writeVInt(numFields); return numFields; } else { writer.reset(vectorsStream); int totalFields = 0; for (DocData dd : pendingDocs) { writer.add(dd.numFields); totalFields += dd.numFields; } writer.finish(); return totalFields; } }
/** Returns a sorted array containing unique field numbers */ private int[] flushFieldNums() throws IOException { SortedSet<Integer> fieldNums = new TreeSet<>(); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { fieldNums.add(fd.fieldNum); } } final int numDistinctFields = fieldNums.size(); assert numDistinctFields > 0; final int bitsRequired = PackedInts.bitsRequired(fieldNums.last()); final int token = (Math.min(numDistinctFields - 1, 0x07) << 5) | bitsRequired; vectorsStream.writeByte((byte) token); if (numDistinctFields - 1 >= 0x07) { vectorsStream.writeVInt(numDistinctFields - 1 - 0x07); } final PackedInts.Writer writer = PackedInts.getWriterNoHeader(vectorsStream, PackedInts.Format.PACKED, fieldNums.size(), bitsRequired, 1); for (Integer fieldNum : fieldNums) { writer.add(fieldNum); } writer.finish(); int[] fns = new int[fieldNums.size()]; int i = 0; for (Integer key : fieldNums) { fns[i++] = key; } return fns; }
@Override public void finish() throws IOException { if (finished) { throw new IllegalStateException("already finished"); } finished = true; CodecUtil.writeFooter(dataOut); String indexFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, Lucene60PointsFormat.INDEX_EXTENSION); // Write index file try (IndexOutput indexOut = writeState.directory.createOutput(indexFileName, writeState.context)) { CodecUtil.writeIndexHeader(indexOut, Lucene60PointsFormat.META_CODEC_NAME, Lucene60PointsFormat.INDEX_VERSION_CURRENT, writeState.segmentInfo.getId(), writeState.segmentSuffix); int count = indexFPs.size(); indexOut.writeVInt(count); for(Map.Entry<String,Long> ent : indexFPs.entrySet()) { FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey()); if (fieldInfo == null) { throw new IllegalStateException("wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos"); } indexOut.writeVInt(fieldInfo.number); indexOut.writeVLong(ent.getValue()); } CodecUtil.writeFooter(indexOut); } }
synchronized private void persist() throws IOException { String fileName = SNAPSHOTS_PREFIX + nextWriteGen; IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(refCounts.size()); for(Entry<Long,Integer> ent : refCounts.entrySet()) { out.writeVLong(ent.getKey()); out.writeVInt(ent.getValue()); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); IOUtils.deleteFilesIgnoringExceptions(dir, fileName); } else { IOUtils.close(out); } } dir.sync(Collections.singletonList(fileName)); if (nextWriteGen > 0) { String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1); // exception OK: likely it didn't exist IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile); } nextWriteGen++; }
private void flushNumTerms(int totalFields) throws IOException { int maxNumTerms = 0; for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { maxNumTerms |= fd.numTerms; } } final int bitsRequired = PackedInts.bitsRequired(maxNumTerms); vectorsStream.writeVInt(bitsRequired); final PackedInts.Writer writer = PackedInts.getWriterNoHeader( vectorsStream, PackedInts.Format.PACKED, totalFields, bitsRequired, 1); for (DocData dd : pendingDocs) { for (FieldData fd : dd.fields) { writer.add(fd.numTerms); } } assert writer.ord() == totalFields - 1; writer.finish(); }
try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size()); for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeVInt(fi.number); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); output.writeVInt(fi.getPointDataDimensionCount()); if (fi.getPointDataDimensionCount() != 0) { output.writeVInt(fi.getPointIndexDimensionCount()); output.writeVInt(fi.getPointNumBytes());
/** * Write a block of data (<code>For</code> format). * * @param data the data to write * @param encoded a buffer to use to encode data * @param out the destination output * @throws IOException If there is a low-level I/O error */ void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException { if (isAllEqual(data)) { out.writeByte((byte) ALL_VALUES_EQUAL); out.writeVInt(data[0]); return; } final int numBits = bitsRequired(data); assert numBits > 0 && numBits <= 32 : numBits; final PackedInts.Encoder encoder = encoders[numBits]; final int iters = iterations[numBits]; assert iters * encoder.byteValueCount() >= BLOCK_SIZE; final int encodedSize = encodedSizes[numBits]; assert iters * encoder.byteBlockCount() >= encodedSize; out.writeByte((byte) numBits); encoder.encode(data, 0, encoded, 0, iters); out.writeBytes(encoded, encodedSize); }
@Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size()); for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeString(fi.name); output.writeVInt(fi.number); byte bits = 0x0; if (fi.hasVectors()) bits |= STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= OMIT_NORMS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); } CodecUtil.writeFooter(output); } }
@Override public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED_NUMERIC); long[] stats = writeValues(field, valuesProducer); int numDocsWithField = Math.toIntExact(stats[0]); long numValues = stats[1]; assert numValues >= numDocsWithField; meta.writeInt(numDocsWithField); if (numValues > numDocsWithField) { long start = data.getFilePointer(); meta.writeLong(start); meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT); long addr = 0; addressesWriter.add(addr); SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { addr += values.docValueCount(); addressesWriter.add(addr); } addressesWriter.finish(); meta.writeLong(data.getFilePointer() - start); } }
entries.writeVInt(si.files().size()); for (String file : si.files()) {