@Override public void flush() throws HyracksDataException { for (int i = 0; i < pWriters.length; ++i) { appenders[i].flush(pWriters[i]); } } }
public void reset(IFrame buffer, boolean clear) throws HyracksDataException { frameTupleAppender.reset(buffer, clear); }
/** * @param writer * @param frameTupleAppender * @param tupleAccessor * @param tStartOffset * @param tEndOffset * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, IFrameTupleAccessor tupleAccessor, int tStartOffset, int tEndOffset) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.append(tupleAccessor, tStartOffset, tEndOffset)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); frameTupleAppender.write(writer, true); if (!frameTupleAppender.append(tupleAccessor, tStartOffset, tEndOffset)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, tEndOffset - tStartOffset); } } return flushedBytes; }
/** * @param writer * @param frameTupleAppender * @param accessor0 * @param tIndex0 * @param accessor1 * @param tIndex1 * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendConcatToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, IFrameTupleAccessor accessor0, int tIndex0, IFrameTupleAccessor accessor1, int tIndex1) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.appendConcat(accessor0, tIndex0, accessor1, tIndex1)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); frameTupleAppender.write(writer, true); if (!frameTupleAppender.appendConcat(accessor0, tIndex0, accessor1, tIndex1)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, accessor0.getTupleLength(tIndex0) + accessor1.getTupleLength(tIndex1)); } } return flushedBytes; }
/** * A util function to append the data to appender. If the appender buffer is full, it will directly flush * to the given writer, which saves the detecting logic in the caller. * It will return the bytes that have been flushed. * * @param writer * @param frameTupleAppender * @param fieldSlots * @param bytes * @param offset * @param length * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendSkipEmptyFieldToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, int[] fieldSlots, byte[] bytes, int offset, int length) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.appendSkipEmptyField(fieldSlots, bytes, offset, length)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); frameTupleAppender.write(writer, true); if (!frameTupleAppender.appendSkipEmptyField(fieldSlots, bytes, offset, length)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, length); } } return flushedBytes; }
/** * @param writer * @param frameTupleAppender * @param accessor * @param tIndex * @param fields * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendProjectionToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, IFrameTupleAccessor accessor, int tIndex, int[] fields) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.appendProjection(accessor, tIndex, fields)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); frameTupleAppender.write(writer, true); if (!frameTupleAppender.appendProjection(accessor, tIndex, fields)) { int fTargetSlotsLength = fields.length * Integer.BYTES; int length = fTargetSlotsLength; for (int i = 0; i < fields.length; ++i) { length += (accessor.getFieldEndOffset(tIndex, fields[i]) - accessor.getFieldStartOffset(tIndex, fields[i])); } throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, length); } } return flushedBytes; }
@Override public int flush(IFrameWriter writer) throws HyracksDataException { outputAppender.reset(outputFrame, true); int maxFrameSize = outputFrame.getFrameSize(); int limit = Math.min(tupleCount, outputLimit); int io = 0; for (int ptr = 0; ptr < limit; ++ptr) { int i = tPointers[ptr * ptrSize + ID_FRAME_ID]; int tStart = tPointers[ptr * ptrSize + ID_TUPLE_START]; int tEnd = tPointers[ptr * ptrSize + ID_TUPLE_END]; bufferManager.getFrame(i, info); inputTupleAccessor.reset(info.getBuffer(), info.getStartOffset(), info.getLength()); int flushed = FrameUtils.appendToWriter(writer, outputAppender, inputTupleAccessor, tStart, tEnd); if (flushed > 0) { maxFrameSize = Math.max(maxFrameSize, flushed); io++; } } maxFrameSize = Math.max(maxFrameSize, outputFrame.getFrameSize()); outputAppender.write(writer, true); if (LOGGER.isTraceEnabled()) { LOGGER.trace( "Flushed records:" + limit + " out of " + tupleCount + "; Flushed through " + (io + 1) + " frames"); } return maxFrameSize; }
public void write() throws HyracksDataException { frameTupleAppender.write(outputWriter, true); }
/** * @param writer * @param frameTupleAppender * @param accessor0 * @param tIndex0 * @param fieldSlots1 * @param bytes1 * @param offset1 * @param dataLen1 * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendConcatToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, IFrameTupleAccessor accessor0, int tIndex0, int[] fieldSlots1, byte[] bytes1, int offset1, int dataLen1) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.appendConcat(accessor0, tIndex0, fieldSlots1, bytes1, offset1, dataLen1)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); frameTupleAppender.write(writer, true); if (!frameTupleAppender.appendConcat(accessor0, tIndex0, fieldSlots1, bytes1, offset1, dataLen1)) { int startOffset0 = accessor0.getTupleStartOffset(tIndex0); int endOffset0 = accessor0.getTupleEndOffset(tIndex0); int length0 = endOffset0 - startOffset0; int slotsLen1 = fieldSlots1.length * Integer.BYTES; int length1 = slotsLen1 + dataLen1; throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, length0 + length1); } } return flushedBytes; }
@Override public void close() throws HyracksDataException { HyracksDataException closeException = null; for (int i = 0; i < pWriters.length; ++i) { if (isWriterOpen[i]) { try { appenders[i].write(pWriters[i], true); } catch (Throwable th) { if (closeException == null) { closeException = HyracksDataException.create(th); } else { closeException.addSuppressed(th); } } finally { try { pWriters[i].close(); } catch (Throwable th) { if (closeException == null) { closeException = HyracksDataException.create(th); } else { closeException.addSuppressed(th); } } } } } if (closeException != null) { throw closeException; } }
/** * @param writer * @param tupleAppender * @param fieldEndOffsets * @param byteArray * @param start * @param size * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender tupleAppender, int[] fieldEndOffsets, byte[] byteArray, int start, int size) throws HyracksDataException { int flushedBytes = 0; if (!tupleAppender.append(fieldEndOffsets, byteArray, start, size)) { flushedBytes = tupleAppender.getBuffer().capacity(); tupleAppender.write(writer, true); if (!tupleAppender.append(fieldEndOffsets, byteArray, start, size)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, size); } } return flushedBytes; }
public void flush() throws HyracksDataException { frameTupleAppender.flush(outputWriter); }
public LocalityAwarePartitionDataWriter(IHyracksTaskContext ctx, IPartitionWriterFactory pwFactory, RecordDescriptor recordDescriptor, ITuplePartitionComputer tpc, int nConsumerPartitions, ILocalityMap localityMap, int senderIndex) throws HyracksDataException { int[] consumerPartitions = localityMap.getConsumers(senderIndex, nConsumerPartitions); pWriters = new IFrameWriter[consumerPartitions.length]; appenders = new IFrameTupleAppender[consumerPartitions.length]; isWriterOpen = new boolean[consumerPartitions.length]; for (int i = 0; i < consumerPartitions.length; ++i) { try { pWriters[i] = pwFactory.createFrameWriter(consumerPartitions[i]); appenders[i] = new FrameTupleAppender(); appenders[i].reset(new VSizeFrame(ctx), true); } catch (IOException e) { throw HyracksDataException.create(e); } } tupleAccessor = new FrameTupleAccessor(recordDescriptor); this.tpc = tpc; }
/** * A util function to append the data to appender. If the appender buffer is full, it will directly flush * to the given writer, which saves the detecting logic in the caller. * It will return the bytes that have been flushed. * * @param writer * @param frameTupleAppender * @param bytes * @param offset * @param length * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, byte[] bytes, int offset, int length) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.append(bytes, offset, length)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); frameTupleAppender.write(writer, true); if (!frameTupleAppender.append(bytes, offset, length)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, length); } } return flushedBytes; }
/** * @param writer * @param frameTupleAppender * @param tupleAccessor * @param tIndex * @return the number of bytes that have been flushed, 0 if not get flushed. * @throws HyracksDataException */ public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, IFrameTupleAccessor tupleAccessor, int tIndex) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.append(tupleAccessor, tIndex)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); frameTupleAppender.write(writer, true); if (!frameTupleAppender.append(tupleAccessor, tIndex)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, tupleAccessor.getTupleLength(tIndex)); } } return flushedBytes; }
public static int appendToWriter(IFrameWriter writer, IFrameTupleAppender frameTupleAppender, IFrameTupleAccessor tupleAccessor, int tIndex, ITracer tracer, String name, long cat, String args) throws HyracksDataException { int flushedBytes = 0; if (!frameTupleAppender.append(tupleAccessor, tIndex)) { flushedBytes = frameTupleAppender.getBuffer().capacity(); long tid = tracer.durationB(name, cat, args); frameTupleAppender.write(writer, true); tracer.durationE(tid, cat, args); if (!frameTupleAppender.append(tupleAccessor, tIndex)) { throw HyracksDataException.create(ErrorCode.TUPLE_CANNOT_FIT_INTO_EMPTY_FRAME, tupleAccessor.getTupleLength(tIndex)); } } return flushedBytes; }