public static void writeMagic(WriteChannel out, boolean align) throws IOException { out.write(MAGIC); if (align) { out.align(); } }
/** * Write the serialized Message metadata, prefixed by the length, to the output Channel. This * ensures that it aligns to an 8 byte boundary and will adjust the message length to include * any padding used for alignment. * * @param out Output Channel * @param messageLength Number of bytes in the message buffer, written as little Endian prefix * @param messageBuffer Message metadata buffer to be written, this does not include any * message body data which should be subsequently written to the Channel * @return Number of bytes written * @throws IOException on error */ public static int writeMessageBuffer(WriteChannel out, int messageLength, ByteBuffer messageBuffer) throws IOException { // ensure that message aligns to 8 byte padding - 4 bytes for size, then message body if ((messageLength + 4) % 8 != 0) { messageLength += 8 - (messageLength + 4) % 8; } out.writeIntLittleEndian(messageLength); out.write(messageBuffer); out.align(); // any bytes written are already captured by our size modification above return messageLength + 4; }
/** * Write the Arrow buffers of the record batch to the output channel. * * @param out the output channel to write the buffers to * @param batch an ArrowRecordBatch containing buffers to be written * @return the number of bytes written * @throws IOException on error */ public static long writeBatchBuffers(WriteChannel out, ArrowRecordBatch batch) throws IOException { long bufferStart = out.getCurrentPosition(); List<ArrowBuf> buffers = batch.getBuffers(); List<ArrowBuffer> buffersLayout = batch.getBuffersLayout(); for (int i = 0; i < buffers.size(); i++) { ArrowBuf buffer = buffers.get(i); ArrowBuffer layout = buffersLayout.get(i); long startPosition = bufferStart + layout.getOffset(); if (startPosition != out.getCurrentPosition()) { out.writeZeros((int) (startPosition - out.getCurrentPosition())); } out.write(buffer); if (out.getCurrentPosition() != startPosition + layout.getSize()) { throw new IllegalStateException("wrong buffer size: " + out.getCurrentPosition() + " != " + startPosition + layout.getSize()); } } out.align(); return out.getCurrentPosition() - bufferStart; }
/** * Serializes an ArrowRecordBatch. Returns the offset and length of the written batch. * * @param out where to write the batch * @param batch the object to serialize to out * @return the serialized block metadata * @throws IOException if something went wrong */ public static ArrowBlock serialize(WriteChannel out, ArrowRecordBatch batch) throws IOException { long start = out.getCurrentPosition(); int bodyLength = batch.computeBodyLength(); assert bodyLength % 8 == 0; FlatBufferBuilder builder = new FlatBufferBuilder(); int batchOffset = batch.writeTo(builder); ByteBuffer serializedMessage = serializeMessage(builder, MessageHeader.RecordBatch, batchOffset, bodyLength); int metadataLength = serializedMessage.remaining(); // calculate alignment bytes so that metadata length points to the correct location after alignment int padding = (int) ((start + metadataLength + 4) % 8); if (padding != 0) { metadataLength += (8 - padding); } out.writeIntLittleEndian(metadataLength); out.write(serializedMessage); // Align the output to 8 byte boundary. out.align(); long bufferLength = writeBatchBuffers(out, batch); assert bufferLength % 8 == 0; // Metadata size in the Block account for the size prefix return new ArrowBlock(start, metadataLength + 4, bufferLength); }