@Override public void addPosition(final int position, final BytesRef payload, final int startOffset, final int endOffset) throws IOException { assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; // we always receive node ids in the payload assert payload != null; // decode payload sirenPayload.decode(payload); final IntsRef node = sirenPayload.getNode(); // check if we received the same node // TODO: we pay the cost of decoding the node before testing the equality // we could instead directly compute the node hash based on the byte array final int nodeHash = node.hashCode(); if (lastNodeHash != nodeHash) { // if different node // add term freq for previous node if not first payload. if (lastNodeHash != Long.MAX_VALUE) { this.addTermFreqInNode(); } // add new node this.addNode(node); } lastNodeHash = nodeHash; // add position this.addPosition(sirenPayload.getPosition()); }
@Override public final boolean incrementToken() throws IOException { if (input.incrementToken()) { // encode node path final BytesRef bytes = codec.encode(nodeAtt.node(), posAtt.position()); payloadAtt.setPayload(bytes); return true; } else { return false; } }
@Override public BytesRef encode(final IntsRef node, final int pos) { // max case : 1 int = 5 bytes this.ensureByteBufferSize((node.length + 1) * 5); bb.clear(); // encode position CodecUtils.vIntToByteArray(pos, bb); // encode node this.setNode(node.ints, node.offset, node.length); final int limit = ints.length - ints.offset; for (int i = ints.offset; i < limit; i++) { CodecUtils.vIntToByteArray(ints.ints[i], bb); } bb.flip(); this.setData(bb.array(), bb.position(), bb.limit()); return bytes; }
@Override public void decode(final BytesRef data) { // max case : 1 byte = 1 int this.ensureIntBufferSize(data.length); ib.clear(); this.setData(data.bytes, data.offset, data.length); // decode position and node final int limit = bytes.length - bytes.offset; while (bytes.offset < limit) { ib.put(CodecUtils.byteArrayToVInt(bytes)); } ib.flip(); // set position pos = ib.get(); // set node this.setNode(ib.array(), ib.position(), ib.limit() - ib.position()); }
@Test public void testRandomVInt2() throws Exception { final Random r = LuceneTestCase.random(); for (int i = 0; i < 10000; i++) { final int value1 = r.nextInt(Integer.MAX_VALUE); final int value2 = r.nextInt(Integer.MAX_VALUE); final IntsRef ints = new IntsRef(new int[] { value1,value2 }, 0, 2); final int pos = r.nextInt(Integer.MAX_VALUE); final BytesRef bytes = codec.encode(ints, pos); codec.decode(bytes); final IntsRef node = codec.getNode(); assertEquals(ints.ints[0], node.ints[node.offset]); assertEquals(ints.ints[1], node.ints[node.offset + 1]); assertEquals(pos, codec.getPosition()); } }
@Test public void testRandomVInt3() throws Exception { final Random r = LuceneTestCase.random(); for (int i = 0; i < 10000; i++) { final int value1 = r.nextInt(Integer.MAX_VALUE); final int value2 = r.nextInt(Integer.MAX_VALUE); final int value3 = r.nextInt(Integer.MAX_VALUE); final IntsRef ints = new IntsRef(new int[] { value1,value2,value3 }, 0, 3); final int pos = r.nextInt(Integer.MAX_VALUE); final BytesRef bytes = codec.encode(ints, pos); codec.decode(bytes); final IntsRef node = codec.getNode(); assertEquals(ints.ints[0], node.ints[node.offset]); assertEquals(ints.ints[1], node.ints[node.offset + 1]); assertEquals(ints.ints[2], node.ints[node.offset + 2]); assertEquals(pos, codec.getPosition()); } }
@Test public void testSimpleVInt() throws Exception { IntsRef ints = new IntsRef(new int[] { 12,43 }, 0, 2); int pos = 256; BytesRef bytes = codec.encode(ints, pos); codec.decode(bytes); IntsRef node = codec.getNode(); assertEquals(ints.ints[0], node.ints[node.offset]); assertEquals(ints.ints[1], node.ints[node.offset + 1]); assertEquals(pos, codec.getPosition()); ints = new IntsRef(new int[] { 3, 2 }, 0, 2); pos = 2; bytes = codec.encode(ints, pos); codec.decode(bytes); node = codec.getNode(); assertEquals(ints.ints[0], node.ints[node.offset]); assertEquals(ints.ints[1], node.ints[node.offset + 1]); assertEquals(pos, codec.getPosition()); ints = new IntsRef(new int[] { 0, 1 }, 0, 2); pos = 0; bytes = codec.encode(ints, pos); codec.decode(bytes); node = codec.getNode(); assertEquals(ints.ints[0], node.ints[node.offset]); assertEquals(ints.ints[1], node.ints[node.offset + 1]); assertEquals(pos, codec.getPosition()); }