void add(long value, int docID) throws IOException { if (valueCount >= maxValuesSortInHeap) { if (writer == null) { switchToOffline(); } scratchBytesOutput.reset(scratchBytes); scratchBytesOutput.writeLong(value); scratchBytesOutput.writeVInt(docID); scratchBytesOutput.writeVLong(valueCount); writer.write(scratchBytes, 0, scratchBytes.length); } else { // Not too many points added yet, continue using heap: heapWriter.append(value, valueCount, docID); } valueCount++; globalMaxValue = Math.max(value, globalMaxValue); globalMinValue = Math.min(value, globalMinValue); }
protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException { if (spare.length + 8 >= buffer.length) { buffer = ArrayUtil.grow(buffer, spare.length + 8); } output.reset(buffer); output.writeBytes(spare.bytes, spare.offset, spare.length); output.writeLong(weight); writer.write(buffer, 0, output.getPosition()); }
/** If the current segment has too many points then we switchover to temp files / offline sort. */ private void switchToOffline() throws IOException { // For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree: tempInput = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "in", ""); writer = new OfflineSorter.ByteSequencesWriter(tempInput); for(int i=0;i<valueCount;i++) { scratchBytesOutput.reset(scratchBytes); scratchBytesOutput.writeLong(heapWriter.values[i]); scratchBytesOutput.writeVInt(heapWriter.docIDs[i]); scratchBytesOutput.writeVLong(i); // TODO: can/should OfflineSorter optimize the fixed-width case? writer.write(scratchBytes, 0, scratchBytes.length); } heapWriter = null; }
/** encodes an entry (bytes+(contexts)+(payload)+weight) to the provided writer */ protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, BytesRef payload, Set<BytesRef> contexts, long weight) throws IOException { int requiredLength = spare.length + 8 + ((hasPayloads) ? 2 + payload.length : 0); if (hasContexts) { for(BytesRef ctx : contexts) { requiredLength += 2 + ctx.length; } requiredLength += 2; // for length of contexts } if (requiredLength >= buffer.length) { buffer = ArrayUtil.grow(buffer, requiredLength); } output.reset(buffer); output.writeBytes(spare.bytes, spare.offset, spare.length); if (hasContexts) { for (BytesRef ctx : contexts) { output.writeBytes(ctx.bytes, ctx.offset, ctx.length); output.writeShort((short) ctx.length); } output.writeShort((short) contexts.size()); } if (hasPayloads) { output.writeBytes(payload.bytes, payload.offset, payload.length); output.writeShort((short) payload.length); } output.writeLong(weight); writer.write(buffer, 0, output.getPosition()); }