/** * Constructs an instance of the BasicDocumentIndexEntry. * @param length * @param pointer */ public BasicDocumentIndexEntry(int length, BitIndexPointer pointer) { doclength = length; bytes = pointer.getOffset(); bits = pointer.getOffsetBits(); bits += pointer.getFileNumber() << FILE_SHIFT; entries = pointer.getNumberOfEntries(); } /**
/** * {@inheritDoc} */ public void setPointer(Pointer p) { bytes = ((BitIndexPointer)p).getOffset(); bits = ((BitIndexPointer)p).getOffsetBits(); }
/** * {@inheritDoc} */ public void setBitIndexPointer(BitIndexPointer pointer) { n_t = pointer.getNumberOfEntries(); setOffset(pointer); startBitOffset += (byte)(pointer.getFileNumber() << FILE_SHIFT); } /**
p1.setNumberOfEntries(entries); assertEquals(entries, p1.getNumberOfEntries()); viaWritable(p1,p2, SIZE); assertEquals(entries, p2.getNumberOfEntries()); p1.setOffset(offset, (byte)0); assertEquals(offset, p1.getOffset()); viaWritable(p1,p2, SIZE); assertEquals(offset, p2.getOffset()); p1.setOffset(0l, bits); p1.setFileNumber(fileId); assertEquals("desired bits="+bits+" file="+fileId, bits, p1.getOffsetBits()); assertEquals("desired bits="+bits+" file="+fileId, fileId, p1.getFileNumber()); p1.setFileNumber(fileId); p1.setOffset(0l, bits); assertEquals("desired bits="+bits+" file="+fileId, bits, p1.getOffsetBits()); assertEquals("desired bits="+bits+" file="+fileId, fileId, p1.getFileNumber()); assertEquals("desired bits="+bits+" file="+fileId, bits, p2.getOffsetBits()); assertEquals("desired bits="+bits+" file="+fileId, fileId, p2.getFileNumber());
if (file.getByteOffset() != pointer.getOffset()) file.skipBytes(pointer.getOffset() - file.getByteOffset()); if (file.getBitOffset() != pointer.getOffsetBits()) file.skipBits(pointer.getOffsetBits() - file.getBitOffset()); final int df = pointer.getNumberOfEntries(); if (fieldCount > 0) { //if there are tag information to process documentTerms = new int[2+fieldCount][df];
@Override public IterablePosting getPostings(Pointer _pointer) throws IOException { BitIndexPointer pointer = (BitIndexPointer)_pointer; ByteIn in = this.file[pointer.getFileNumber()].readReset(pointer.getOffset()); DocumentIndex fixedDi = pointer instanceof DocumentIndexEntry ? new PostingIndex.DocidSpecificDocumentIndex(documentIndex, (DocumentIndexEntry)pointer) : documentIndex; if (hasBlocks > 0) if (fieldsCount > 0) return new BlockFieldIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, fieldsCount, hasBlocks, maxBlocks, idsCodec, tfsCodec, fieldsCodec, blocksCodec); else return new BlockIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, hasBlocks, maxBlocks, idsCodec, tfsCodec, blocksCodec); else if (fieldsCount > 0) return new FieldIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, fieldsCount, idsCodec, tfsCodec, fieldsCodec); else return new BasicIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, idsCodec, tfsCodec); // return new IntegerCodingIterablePosting(in, // lEntry.getNumberOfEntries(), documentIndex, chunkSize, // fieldsCount, hasBlocks, idsCodec, tfsCodec, fieldsCodec, // blocksCodec); }
protected BitIndexPointer _next() { if (!pointerList.hasNext()) return null; entriesSkipped = 0; BitIndexPointer pointer = (BitIndexPointer) pointerList.next(); while (pointer.getNumberOfEntries() == 0) { entriesSkipped++; if (pointerList.hasNext()) { pointer = (BitIndexPointer) pointerList.next(); } else { return null; } } return pointer; }
@Override public IterablePosting getPostings(Pointer pointer) throws IOException { final BitIn _file = this.file[((BitIndexPointer)pointer).getFileNumber()].readReset(((BitIndexPointer)pointer).getOffset(), ((BitIndexPointer)pointer).getOffsetBits()); IterablePosting rtr = null; try{ rtr = (fieldCount > 0) ? postingConstructor.newInstance(_file, pointer.getNumberOfEntries(), doi, fieldCount) : postingConstructor.newInstance(_file, pointer.getNumberOfEntries(), doi); } catch (Exception e) { throw new WrappedIOException(e); } return rtr; } /**
/** Write out the specified postings, but allowing the delta for the first document to be adjusted * @param iterator an Iterator of Posting objects * @param previousId id of the previous posting in this stream */ @Override public BitIndexPointer writePostings(Iterator<Posting> iterator, int previousId) throws IOException { BitIndexPointer pointer = new SimpleBitIndexPointer(); pointer.setOffset(output.getByteOffset(), output.getBitOffset()); int numberOfEntries = 0; Posting posting = null; while(iterator.hasNext()) { posting = iterator.next(); output.writeGamma(posting.getId() - previousId); lastDocid = previousId = posting.getId(); writePostingNotDocid(posting); numberOfEntries++; } pointer.setNumberOfEntries(numberOfEntries); return pointer; }
p.setOffset(file.getOffset()); p.setNumberOfEntries(ids.length); p.write(dos);
protected static void viaWritable(final BitIndexPointer p1, final BitIndexPointer p2, final int SIZE) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); p1.write(dos); assertEquals(SIZE, baos.size()); DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray())); p2.readFields(dis); }
pointer.setOffset(output.getByteOffset(), output.getBitOffset());
pin.readFields(dis); value.setPointer(pin); los.writeNextEntry(lee.getKey(), value);
if (file.getByteOffset() != pointer.getOffset()) file.skipBytes(pointer.getOffset() - file.getByteOffset()); if (file.getBitOffset() != pointer.getOffsetBits()) file.skipBits(pointer.getOffsetBits() - file.getBitOffset()); final int df = pointer.getNumberOfEntries(); if (fieldCount > 0) { //if there are tag information to process documentTerms = new int[2+fieldCount][df];
@Override public IterablePosting getPostings(Pointer _pointer) throws IOException { BitIndexPointer pointer = (BitIndexPointer)_pointer; ByteIn in = this.file[pointer.getFileNumber()].readReset(pointer.getOffset()); DocumentIndex fixedDi = pointer instanceof DocumentIndexEntry ? new PostingIndex.DocidSpecificDocumentIndex(documentIndex, (DocumentIndexEntry)pointer) : documentIndex; if (hasBlocks > 0) if (fieldsCount > 0) return new BlockFieldIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, fieldsCount, hasBlocks, maxBlocks, idsCodec, tfsCodec, fieldsCodec, blocksCodec); else return new BlockIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, hasBlocks, maxBlocks, idsCodec, tfsCodec, blocksCodec); else if (fieldsCount > 0) return new FieldIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, fieldsCount, idsCodec, tfsCodec, fieldsCodec); else return new BasicIntegerCodingIterablePosting(in, pointer.getNumberOfEntries(), fixedDi, chunkSize, idsCodec, tfsCodec); // return new IntegerCodingIterablePosting(in, // lEntry.getNumberOfEntries(), documentIndex, chunkSize, // fieldsCount, hasBlocks, idsCodec, tfsCodec, fieldsCodec, // blocksCodec); }
/** * {@inheritDoc} */ public void setBitIndexPointer(BitIndexPointer pointer) { entries = pointer.getNumberOfEntries(); bits = 0; setOffset(pointer); bits += pointer.getFileNumber() << FILE_SHIFT; } /**
protected BitIndexPointer _next() { if (!pointerList.hasNext()) return null; entriesSkipped = 0; BitIndexPointer pointer = (BitIndexPointer) pointerList.next(); while (pointer.getNumberOfEntries() == 0) { entriesSkipped++; if (pointerList.hasNext()) { pointer = (BitIndexPointer) pointerList.next(); } else { return null; } } return pointer; }
@Override public IterablePosting getPostings(Pointer pointer) throws IOException { final BitIn _file = this.file[((BitIndexPointer)pointer).getFileNumber()].readReset(((BitIndexPointer)pointer).getOffset(), ((BitIndexPointer)pointer).getOffsetBits()); IterablePosting rtr = null; try{ rtr = (fieldCount > 0) ? postingConstructor.newInstance(_file, pointer.getNumberOfEntries(), doi, fieldCount) : postingConstructor.newInstance(_file, pointer.getNumberOfEntries(), doi); } catch (Exception e) { throw new WrappedIOException(e); } return rtr; } /**
/** * {@inheritDoc} */ public void setPointer(Pointer p) { bytes = ((BitIndexPointer)p).getOffset(); bits = ((BitIndexPointer)p).getOffsetBits(); }
/** Write out the specified postings, but allowing the delta for the first document to be adjusted * @param postings IterablePosting postings accessed through an IterablePosting object * @param previousId id of the previous posting in this stream */ @Override public BitIndexPointer writePostings(IterablePosting postings, int previousId) throws IOException { BitIndexPointer pointer = new SimpleBitIndexPointer(); pointer.setOffset(output.getByteOffset(), output.getBitOffset()); int numberOfEntries = 0; while(postings.next() != IterablePosting.EOL) { output.writeGamma(postings.getId() - previousId); //System.err.println("Writing id" + postings.getId()); lastDocid = previousId = postings.getId(); writePostingNotDocid(postings); numberOfEntries++; } pointer.setNumberOfEntries(numberOfEntries); return pointer; }