@Override protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { int delta = curDoc - lastSkipDoc[level]; skipBuffer.writeVInt(delta); lastSkipDoc[level] = curDoc; skipBuffer.writeVLong(curDocPointer - lastSkipDocPointer[level]); lastSkipDocPointer[level] = curDocPointer; if (fieldHasPositions) { skipBuffer.writeVLong(curPosPointer - lastSkipPosPointer[level]); lastSkipPosPointer[level] = curPosPointer; skipBuffer.writeVInt(curPosBufferUpto); if (fieldHasPayloads) { skipBuffer.writeVInt(curPayloadByteUpto); } if (fieldHasOffsets || fieldHasPayloads) { skipBuffer.writeVLong(curPayPointer - lastSkipPayPointer[level]); lastSkipPayPointer[level] = curPayPointer; } } } }
@Override public void finish(FieldInfos fis, int numDocs) throws IOException { if (!pendingDocs.isEmpty()) { flush(); numDirtyChunks++; // incomplete: we had to force this flush } if (numDocs != this.numDocs) { throw new RuntimeException("Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs); } indexWriter.finish(numDocs, vectorsStream.getFilePointer()); vectorsStream.writeVLong(numChunks); vectorsStream.writeVLong(numDirtyChunks); CodecUtil.writeFooter(vectorsStream); }
void finish(int numDocs, long maxPointer) throws IOException { if (numDocs != totalDocs) { throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs); } if (blockChunks > 0) { writeBlock(); } fieldsIndexOut.writeVInt(0); // end marker fieldsIndexOut.writeVLong(maxPointer); CodecUtil.writeFooter(fieldsIndexOut); }
@Override public void finish(FieldInfos fis, int numDocs) throws IOException { if (numBufferedDocs > 0) { flush(); numDirtyChunks++; // incomplete: we had to force this flush } else { assert bufferedDocs.getPosition() == 0; } if (docBase != numDocs) { throw new RuntimeException("Wrote " + docBase + " docs, finish called with numDocs=" + numDocs); } indexWriter.finish(numDocs, fieldsStream.getFilePointer()); fieldsStream.writeVLong(numChunks); fieldsStream.writeVLong(numDirtyChunks); CodecUtil.writeFooter(fieldsStream); assert bufferedDocs.getPosition() == 0; }
@Override public void finish() throws IOException { if (finished) { throw new IllegalStateException("already finished"); } finished = true; CodecUtil.writeFooter(dataOut); String indexFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, Lucene60PointsFormat.INDEX_EXTENSION); // Write index file try (IndexOutput indexOut = writeState.directory.createOutput(indexFileName, writeState.context)) { CodecUtil.writeIndexHeader(indexOut, Lucene60PointsFormat.META_CODEC_NAME, Lucene60PointsFormat.INDEX_VERSION_CURRENT, writeState.segmentInfo.getId(), writeState.segmentSuffix); int count = indexFPs.size(); indexOut.writeVInt(count); for(Map.Entry<String,Long> ent : indexFPs.entrySet()) { FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey()); if (fieldInfo == null) { throw new IllegalStateException("wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos"); } indexOut.writeVInt(fieldInfo.number); indexOut.writeVLong(ent.getValue()); } CodecUtil.writeFooter(indexOut); } }
synchronized private void persist() throws IOException { String fileName = SNAPSHOTS_PREFIX + nextWriteGen; IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(refCounts.size()); for(Entry<Long,Integer> ent : refCounts.entrySet()) { out.writeVLong(ent.getKey()); out.writeVInt(ent.getValue()); } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(out); IOUtils.deleteFilesIgnoringExceptions(dir, fileName); } else { IOUtils.close(out); } } dir.sync(Collections.singletonList(fileName)); if (nextWriteGen > 0) { String lastSaveFile = SNAPSHOTS_PREFIX + (nextWriteGen-1); // exception OK: likely it didn't exist IOUtils.deleteFilesIgnoringExceptions(dir, lastSaveFile); } nextWriteGen++; }
/** * Writes the buffered skip lists to the given output. * * @param output the IndexOutput the skip lists shall be written to * @return the pointer the skip list starts */ public long writeSkip(IndexOutput output) throws IOException { long skipPointer = output.getFilePointer(); //System.out.println("skipper.writeSkip fp=" + skipPointer); if (skipBuffer == null || skipBuffer.length == 0) return skipPointer; for (int level = numberOfSkipLevels - 1; level > 0; level--) { long length = skipBuffer[level].getFilePointer(); if (length > 0) { output.writeVLong(length); skipBuffer[level].writeTo(output); } } skipBuffer[0].writeTo(output); return skipPointer; } }
termsOut.writeVLong(field.numTerms); termsOut.writeVInt(field.rootCode.length); termsOut.writeBytes(field.rootCode.bytes, field.rootCode.offset, field.rootCode.length); assert field.fieldInfo.getIndexOptions() != IndexOptions.NONE; if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS) { termsOut.writeVLong(field.sumTotalTermFreq); termsOut.writeVLong(field.sumDocFreq); termsOut.writeVInt(field.docCount); termsOut.writeVInt(field.longsSize); indexOut.writeVLong(field.indexStartFP); writeBytesRef(termsOut, field.minTerm); writeBytesRef(termsOut, field.maxTerm);
private void writeIndex(IndexOutput out, int countPerLeaf, int numLeaves, byte[] packedIndex) throws IOException { CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT); out.writeVInt(numDataDims); out.writeVInt(numIndexDims); out.writeVInt(countPerLeaf); out.writeVInt(bytesPerDim); assert numLeaves > 0; out.writeVInt(numLeaves); out.writeBytes(minPackedValue, 0, packedIndexBytesLength); out.writeBytes(maxPackedValue, 0, packedIndexBytesLength); out.writeVLong(pointCount); out.writeVInt(docsSeen.cardinality()); out.writeVInt(packedIndex.length); out.writeBytes(packedIndex, 0, packedIndex.length); }
out.writeVLong(counter); // write counter out.writeInt(size());
fieldsIndexOut.writeVLong(firstStartPointer); final long avgChunkSize; if (blockChunks == 1) { avgChunkSize = (maxStartPointer - firstStartPointer) / (blockChunks - 1); fieldsIndexOut.writeVLong(avgChunkSize); long startPointer = 0; maxDelta = 0;
private void addTermsDict(SortedSetDocValues values) throws IOException { final long size = values.getValueCount(); meta.writeVLong(size); meta.writeInt(Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT);
public void write(final IndexOutput indexOut, final boolean absolute) throws IOException { // logger.debug("Write index at {}", fp); if (absolute) { indexOut.writeVLong(fp); } else { indexOut.writeVLong(fp - lastFP); } lastFP = fp; }
@Override public void finish(FieldInfos fis, int numDocs) throws IOException { if (!pendingDocs.isEmpty()) { flush(); numDirtyChunks++; // incomplete: we had to force this flush } if (numDocs != this.numDocs) { throw new RuntimeException("Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs); } indexWriter.finish(numDocs, vectorsStream.getFilePointer()); vectorsStream.writeVLong(numChunks); vectorsStream.writeVLong(numDirtyChunks); CodecUtil.writeFooter(vectorsStream); }
@Override public void finish(FieldInfos fis, int numDocs) throws IOException { if (!pendingDocs.isEmpty()) { flush(); numDirtyChunks++; // incomplete: we had to force this flush } if (numDocs != this.numDocs) { throw new RuntimeException("Wrote " + this.numDocs + " docs, finish called with numDocs=" + numDocs); } indexWriter.finish(numDocs, vectorsStream.getFilePointer()); vectorsStream.writeVLong(numChunks); vectorsStream.writeVLong(numDirtyChunks); CodecUtil.writeFooter(vectorsStream); }
void finish(int numDocs, long maxPointer) throws IOException { if (numDocs != totalDocs) { throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs); } if (blockChunks > 0) { writeBlock(); } fieldsIndexOut.writeVInt(0); // end marker fieldsIndexOut.writeVLong(maxPointer); CodecUtil.writeFooter(fieldsIndexOut); }
void finish(int numDocs, long maxPointer) throws IOException { if (numDocs != totalDocs) { throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs); } if (blockChunks > 0) { writeBlock(); } fieldsIndexOut.writeVInt(0); // end marker fieldsIndexOut.writeVLong(maxPointer); CodecUtil.writeFooter(fieldsIndexOut); }
void finish(int numDocs, long maxPointer) throws IOException { if (numDocs != totalDocs) { throw new IllegalStateException("Expected " + numDocs + " docs, but got " + totalDocs); } if (blockChunks > 0) { writeBlock(); } fieldsIndexOut.writeVInt(0); // end marker fieldsIndexOut.writeVLong(maxPointer); CodecUtil.writeFooter(fieldsIndexOut); }
public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { if (termCount > 0) { out.writeVInt(termCount); out.writeVInt(field.number); if (field.getIndexOptions() != IndexOptions.DOCS) { out.writeVLong(sumTotalTermFreq); } out.writeVLong(sumDocFreq); out.writeVInt(docCount); FST<BytesRef> fst = builder.finish(); fst.save(out); //System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer()); } } }
private void flushBuffer() throws IOException { if (_bufferPosition > 0) { _compressor.reset(); _compressor.setInput(_buffer, 0, _bufferPosition); _compressor.finish(); long filePointer = _output.getFilePointer(); int length = _compressor.compress(_compressedBuffer, 0, _compressedBuffer.length); _tmpOutput.writeVLong(filePointer); _tmpOutput.writeVInt(length); _blockCount++; _output.writeBytes(_compressedBuffer, 0, length); } }