/** * Writes a codec footer, which records both a checksum * algorithm ID and a checksum. This footer can * be parsed and validated with * {@link #checkFooter(ChecksumIndexInput) checkFooter()}. * <p> * CodecFooter --> Magic,AlgorithmID,Checksum * <ul> * <li>Magic --> {@link DataOutput#writeInt Uint32}. This * identifies the start of the footer. It is always {@value #FOOTER_MAGIC}. * <li>AlgorithmID --> {@link DataOutput#writeInt Uint32}. This * indicates the checksum algorithm used. Currently this is always 0, * for zlib-crc32. * <li>Checksum --> {@link DataOutput#writeLong Uint64}. The * actual checksum value for all previous bytes in the stream, including * the bytes from Magic and AlgorithmID. * </ul> * * @param out Output stream * @throws IOException If there is an I/O error writing to the underlying medium. */ public static void writeFooter(IndexOutput out) throws IOException { out.writeInt(FOOTER_MAGIC); out.writeInt(0); writeCRC(out); }
@Override public void append(byte[] packedValue, long ord, int docID) throws IOException { assert packedValue.length == packedBytesLength; out.writeBytes(packedValue, 0, packedValue.length); out.writeInt(docID); if (singleValuePerDoc == false) { if (longOrds) { out.writeLong(ord); } else { assert ord <= Integer.MAX_VALUE; out.writeInt((int) ord); } } count++; assert expectedCount == 0 || count <= expectedCount; }
@Override public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.NUMERIC); writeValues(field, new EmptyDocValuesProducer() { @Override public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { return DocValues.singleton(valuesProducer.getNumeric(field)); } }); }
@Override public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED); doAddSortedField(field, valuesProducer); }
@Override public void close() throws IOException { boolean success = false; try { if (meta != null) { meta.writeInt(-1); // write EOF marker CodecUtil.writeFooter(meta); // write checksum } if (data != null) { CodecUtil.writeFooter(data); // write checksum } success = true; } finally { if (success) { IOUtils.close(data, meta); } else { IOUtils.closeWhileHandlingException(data, meta); } meta = data = null; } }
@Override public void close() throws IOException { boolean success = false; try { if (meta != null) { meta.writeInt(-1); // write EOF marker CodecUtil.writeFooter(meta); // write checksum } if (data != null) { CodecUtil.writeFooter(data); // write checksum } success = true; } finally { if (success) { IOUtils.close(data, meta); } else { IOUtils.closeWhileHandlingException(data, meta); } meta = data = null; } }
private void writeValues(NumericDocValues values, int numBytesPerValue, IndexOutput out) throws IOException, AssertionError { for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { long value = values.longValue(); switch (numBytesPerValue) { case 1: out.writeByte((byte) value); break; case 2: out.writeShort((short) value); break; case 4: out.writeInt((int) value); break; case 8: out.writeLong(value); break; default: throw new AssertionError(); } } } }
vectorsStream.writeInt(Float.floatToRawIntBits(charsPerTerm[i]));
meta.writeInt(Float.floatToIntBits(avgInc)); meta.writeLong(data.getFilePointer() - baseDataPointer); if (maxDelta == 0) {
@Override public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED_NUMERIC); long[] stats = writeValues(field, valuesProducer); int numDocsWithField = Math.toIntExact(stats[0]); long numValues = stats[1]; assert numValues >= numDocsWithField; meta.writeInt(numDocsWithField); if (numValues > numDocsWithField) { long start = data.getFilePointer(); meta.writeLong(start); meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT); final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT); long addr = 0; addressesWriter.add(addr); SortedNumericDocValues values = valuesProducer.getSortedNumeric(field); for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) { addr += values.docValueCount(); addressesWriter.add(addr); } addressesWriter.finish(); meta.writeLong(data.getFilePointer() - start); } }
out.writeInt(size()); throw new IllegalStateException("cannot write segment: invalid maxDoc segment=" + si.name + " maxDoc=" + si.maxDoc() + " delCount=" + delCount); out.writeInt(delCount); out.writeLong(siPerCommit.getFieldInfosGen()); out.writeLong(siPerCommit.getDocValuesGen()); throw new IllegalStateException("cannot write segment: invalid maxDoc segment=" + si.name + " maxDoc=" + si.maxDoc() + " softDelCount=" + softDelCount); out.writeInt(softDelCount); out.writeSetOfStrings(siPerCommit.getFieldInfosFiles()); final Map<Integer,Set<String>> dvUpdatesFiles = siPerCommit.getDocValuesUpdatesFiles(); out.writeInt(dvUpdatesFiles.size()); for (Entry<Integer,Set<String>> e : dvUpdatesFiles.entrySet()) { out.writeInt(e.getKey()); out.writeSetOfStrings(e.getValue());
if (min >= max) { numBitsPerValue = 0; meta.writeInt(-1); } else { if (uniqueValues != null final Long[] sortedUniqueValues = uniqueValues.toArray(new Long[0]); Arrays.sort(sortedUniqueValues); meta.writeInt(sortedUniqueValues.length); for (Long v : sortedUniqueValues) { meta.writeLong(v); if (doBlocks) { numBitsPerValue = 0xFF; meta.writeInt(-2 - NUMERIC_BLOCK_SHIFT); } else { numBitsPerValue = DirectWriter.unsignedBitsRequired((max - min) / gcd); min = 0; meta.writeInt(-1);
private void writeBlock(long[] values, int length, long gcd, GrowableByteArrayDataOutput buffer) throws IOException { assert length > 0; long min = values[0]; long max = values[0]; for (int i = 1; i < length; ++i) { final long v = values[i]; assert Math.floorMod(values[i] - min, gcd) == 0; min = Math.min(min, v); max = Math.max(max, v); } if (min == max) { data.writeByte((byte) 0); data.writeLong(min); } else { final int bitsPerValue = DirectWriter.unsignedBitsRequired(max - min); buffer.reset(); assert buffer.getPosition() == 0; final DirectWriter w = DirectWriter.getInstance(buffer, length, bitsPerValue); for (int i = 0; i < length; ++i) { w.add((values[i] - min) / gcd); } w.finish(); data.writeByte((byte) bitsPerValue); data.writeLong(min); data.writeInt(buffer.getPosition()); data.writeBytes(buffer.getBytes(), buffer.getPosition()); } }
private void addTermsDict(SortedSetDocValues values) throws IOException { final long size = values.getValueCount(); meta.writeVLong(size); meta.writeInt(Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT); meta.writeInt(DIRECT_MONOTONIC_BLOCK_SHIFT); long numBlocks = (size + Lucene70DocValuesFormat.TERMS_DICT_BLOCK_MASK) >>> Lucene70DocValuesFormat.TERMS_DICT_BLOCK_SHIFT; DirectMonotonicWriter writer = DirectMonotonicWriter.getInstance(meta, addressBuffer, numBlocks, DIRECT_MONOTONIC_BLOCK_SHIFT); meta.writeInt(maxLength); meta.writeLong(start); meta.writeLong(data.getFilePointer() - start);
@Override public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.BINARY); meta.writeInt(numDocsWithField); meta.writeInt(minLength); meta.writeInt(maxLength); if (maxLength > minLength) { start = data.getFilePointer();
private void writeTermsIndex(SortedSetDocValues values) throws IOException { final long size = values.getValueCount(); meta.writeInt(Lucene70DocValuesFormat.TERMS_DICT_REVERSE_INDEX_SHIFT); long start = data.getFilePointer();
@Override public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException { meta.writeInt(field.number); meta.writeByte(Lucene70DocValuesFormat.SORTED_SET); meta.writeLong(data.getFilePointer() - start); meta.writeInt(numDocsWithField); start = data.getFilePointer(); meta.writeLong(start);
meta.writeInt(numDocsWithField); if (values.getValueCount() <= 1) { meta.writeByte((byte) 0);