/** * If the given {@link FixedBitSet} is large enough to hold {@code numBits+1}, * returns the given bits, otherwise returns a new {@link FixedBitSet} which * can hold the requested number of bits. * <p> * <b>NOTE:</b> the returned bitset reuses the underlying {@code long[]} of * the given {@code bits} if possible. Also, calling {@link #length()} on the * returned bits may return a value greater than {@code numBits}. */ public static FixedBitSet ensureCapacity(FixedBitSet bits, int numBits) { if (numBits < bits.numBits) { return bits; } else { // Depends on the ghost bits being clear! // (Otherwise, they may become visible in the new instance) int numWords = bits2words(numBits); long[] arr = bits.getBits(); if (numWords >= arr.length) { arr = ArrayUtil.grow(arr, numWords + 1); } return new FixedBitSet(arr, arr.length << 6); } }
private static void flush(int block, FixedBitSet buffer, int cardinality, IndexOutput out) throws IOException { assert block >= 0 && block < 65536; out.writeShort((short) block); assert cardinality > 0 && cardinality <= 65536; out.writeShort((short) (cardinality - 1)); if (cardinality > MAX_ARRAY_LENGTH) { if (cardinality != 65536) { // all docs are set for (long word : buffer.getBits()) { out.writeLong(word); } } } else { BitSetIterator it = new BitSetIterator(buffer, cardinality); for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) { out.writeShort((short) doc); } } }
protected int bitLength() { return bitSet.getBits().length; }
/** * Writes vector to a string of the form 010 etc. (no delimiters). * * No terminating newline or delimiter. */ public String writeLongToString() { StringBuilder builder = new StringBuilder(); for (int i = 0; i < (bitSet.getBits().length); ++i) { builder.append(Long.toString(bitSet.getBits()[i])+"|"); } return builder.toString(); }
private long docsWithFieldBytesUsed() { // size of the long[] + some overhead return RamUsageEstimator.sizeOf(docsWithField.getBits()) + 64; }
private long docsWithFieldBytesUsed() { // size of the long[] + some overhead return RamUsageEstimator.sizeOf(docsWithField.getBits()) + 64; }
@Override public long ramBytesUsed() { return RamUsageEstimator.sizeOf(filter.getBits()); }
private long docsWithFieldBytesUsed() { // size of the long[] + some overhead return RamUsageEstimator.sizeOf(docsWithField.getBits()) + 64; }
private long docsWithFieldBytesUsed() { // size of the long[] + some overhead return RamUsageEstimator.sizeOf(docsWithField.getBits()) + 64; }
@Override /** * Writes vector out to object output stream. Converts to dense format if necessary. */ public void writeToLuceneStream(IndexOutput outputStream) { if (isSparse) { elementalToSemantic(); } long[] bitArray = bitSet.getBits(); for (int i = 0; i < bitArray.length; i++) { try { outputStream.writeLong(bitArray[i]); } catch (IOException e) { logger.severe("Couldn't write binary vector to lucene output stream."); e.printStackTrace(); } } }
/** * Writes vector out to object output stream. Converts to dense format if necessary. Truncates to length k. */ public void writeToLuceneStream(IndexOutput outputStream, int k) { if (isSparse) { elementalToSemantic(); } long[] bitArray = bitSet.getBits(); for (int i = 0; i < k/64; i++) { try { outputStream.writeLong(bitArray[i]); } catch (IOException e) { logger.severe("Couldn't write binary vector to lucene output stream."); e.printStackTrace(); } } }
/** * If the given {@link FixedBitSet} is large enough to hold {@code numBits+1}, * returns the given bits, otherwise returns a new {@link FixedBitSet} which * can hold the requested number of bits. * <p> * <b>NOTE:</b> the returned bitset reuses the underlying {@code long[]} of * the given {@code bits} if possible. Also, calling {@link #length()} on the * returned bits may return a value greater than {@code numBits}. */ public static FixedBitSet ensureCapacity(FixedBitSet bits, int numBits) { if (numBits < bits.numBits) { return bits; } else { // Depends on the ghost bits being clear! // (Otherwise, they may become visible in the new instance) int numWords = bits2words(numBits); long[] arr = bits.getBits(); if (numWords >= arr.length) { arr = ArrayUtil.grow(arr, numWords + 1); } return new FixedBitSet(arr, arr.length << 6); } }
/** * If the given {@link FixedBitSet} is large enough to hold {@code numBits+1}, * returns the given bits, otherwise returns a new {@link FixedBitSet} which * can hold the requested number of bits. * <p> * <b>NOTE:</b> the returned bitset reuses the underlying {@code long[]} of * the given {@code bits} if possible. Also, calling {@link #length()} on the * returned bits may return a value greater than {@code numBits}. */ public static FixedBitSet ensureCapacity(FixedBitSet bits, int numBits) { if (numBits < bits.numBits) { return bits; } else { // Depends on the ghost bits being clear! // (Otherwise, they may become visible in the new instance) int numWords = bits2words(numBits); long[] arr = bits.getBits(); if (numWords >= arr.length) { arr = ArrayUtil.grow(arr, numWords + 1); } return new FixedBitSet(arr, arr.length << 6); } }
/** * If the given {@link FixedBitSet} is large enough to hold {@code numBits+1}, * returns the given bits, otherwise returns a new {@link FixedBitSet} which * can hold the requested number of bits. * <p> * <b>NOTE:</b> the returned bitset reuses the underlying {@code long[]} of * the given {@code bits} if possible. Also, calling {@link #length()} on the * returned bits may return a value greater than {@code numBits}. */ public static FixedBitSet ensureCapacity(FixedBitSet bits, int numBits) { if (numBits < bits.numBits) { return bits; } else { // Depends on the ghost bits being clear! // (Otherwise, they may become visible in the new instance) int numWords = bits2words(numBits); long[] arr = bits.getBits(); if (numWords >= arr.length) { arr = ArrayUtil.grow(arr, numWords + 1); } return new FixedBitSet(arr, arr.length << 6); } }
theSuperpositionSeed += votingRecord.get(q).getBits()[0];
/** * Permute the long[] array underlying the FixedBitSet binary representation */ public void permute(int[] permutation) { if (permutation.length != getDimension() / 64) { throw new IllegalArgumentException("Binary vector of dimension " + getDimension() + " must have permutation of length " + getDimension() / 64 + " not " + permutation.length); } //TODO permute in place without creating additional long[] (if proves problematic at scale) long[] coordinates = bitSet.getBits(); long[] newCoordinates = new long[coordinates.length]; for (int i = 0; i < coordinates.length; ++i) { int positionToAdd = i; positionToAdd = permutation[positionToAdd]; newCoordinates[i] = coordinates[positionToAdd]; } bitSet = new FixedBitSet(newCoordinates, getDimension()); }
long[] bits = filter.getBits(); out.writeInt(bits.length); for (int i = 0; i < bits.length; i++) {
private static void flush(int block, FixedBitSet buffer, int cardinality, IndexOutput out) throws IOException { assert block >= 0 && block < 65536; out.writeShort((short) block); assert cardinality > 0 && cardinality <= 65536; out.writeShort((short) (cardinality - 1)); if (cardinality > MAX_ARRAY_LENGTH) { if (cardinality != 65536) { // all docs are set for (long word : buffer.getBits()) { out.writeLong(word); } } } else { BitSetIterator it = new BitSetIterator(buffer, cardinality); for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) { out.writeShort((short) doc); } } }
@Override public void writeLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException { long gen = info.getNextDelGen(); String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen); FixedBitSet fbs = (FixedBitSet) bits; if (fbs.length() - fbs.cardinality() != info.getDelCount() + newDelCount) { throw new CorruptIndexException("bits.deleted=" + (fbs.length() - fbs.cardinality()) + " info.delcount=" + info.getDelCount() + " newdelcount=" + newDelCount, name); } long data[] = fbs.getBits(); try (IndexOutput output = dir.createOutput(name, context)) { CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), Long.toString(gen, Character.MAX_RADIX)); for (int i = 0; i < data.length; i++) { output.writeLong(data[i]); } CodecUtil.writeFooter(output); } }
@Override public void writeLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException { long gen = info.getNextDelGen(); String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen); FixedBitSet fbs = (FixedBitSet) bits; if (fbs.length() - fbs.cardinality() != info.getDelCount() + newDelCount) { throw new CorruptIndexException("bits.deleted=" + (fbs.length() - fbs.cardinality()) + " info.delcount=" + info.getDelCount() + " newdelcount=" + newDelCount, name); } long data[] = fbs.getBits(); try (IndexOutput output = dir.createOutput(name, context)) { CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), Long.toString(gen, Character.MAX_RADIX)); for (int i = 0; i < data.length; i++) { output.writeLong(data[i]); } CodecUtil.writeFooter(output); } }