private void appendMessage(ByteBuffer message) { int messageLength = message.limit() - message.position(); System.arraycopy(message.array(), message.position(), array, tupleDataEndOffset, messageLength); tupleDataEndOffset += messageLength; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); }
protected void reset(ByteBuffer buffer, boolean clear) { array = buffer.array(); if (clear) { IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), 0); tupleCount = 0; tupleDataEndOffset = FrameConstants.TUPLE_START_OFFSET; } else { tupleCount = IntSerDeUtils.getInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize())); tupleDataEndOffset = tupleCount == 0 ? FrameConstants.TUPLE_START_OFFSET : IntSerDeUtils.getInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - tupleCount * FrameConstants.SIZE_LEN); } }
@Override public void reset(IFrame frame, boolean clear) throws HyracksDataException { super.reset(frame, clear); this.tupleCountOffset = FrameHelper.getTupleCountOffset(frame.getFrameSize()); } }
protected boolean hasEnoughSpace(int fieldCount, int tupleLength) { return tupleDataEndOffset + FrameHelper.calcRequiredSpace(fieldCount, tupleLength) + tupleCount * FrameConstants.SIZE_LEN <= FrameHelper.getTupleCountOffset(frame.getFrameSize()); }
private void appendNullMessage() { array[tupleDataEndOffset] = NULL_FEED_MESSAGE; tupleDataEndOffset++; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); }
@Override public boolean append(byte[] bytes, int offset, int length) throws HyracksDataException { if (canHoldNewTuple(0, length)) { System.arraycopy(bytes, offset, getBuffer().array(), tupleDataEndOffset, length); tupleDataEndOffset += length; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true; } return false; }
/** * Cancels the lastly performed append operation. i.e. decreases the tuple count and resets the data end offset. */ @Override public boolean cancelAppend() throws HyracksDataException { // Decreases tupleCount by one. tupleCount = IntSerDeUtils.getInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize())); if (tupleCount == 0) { // There is no inserted tuple in the given frame. This should not happen. return false; } tupleCount = tupleCount - 1; // Resets tupleCount and DataEndOffset. IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); tupleDataEndOffset = tupleCount == 0 ? FrameConstants.TUPLE_START_OFFSET : IntSerDeUtils.getInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - tupleCount * FrameConstants.SIZE_LEN); return true; }
@Override public boolean append(IFrameTupleAccessor tupleAccessor, int tStartOffset, int tEndOffset) throws HyracksDataException { int length = tEndOffset - tStartOffset; if (canHoldNewTuple(0, length)) { ByteBuffer src = tupleAccessor.getBuffer(); System.arraycopy(src.array(), tStartOffset, array, tupleDataEndOffset, length); tupleDataEndOffset += length; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true; } return false; }
@Override public boolean appendSkipEmptyField(int[] fieldSlots, byte[] bytes, int offset, int length) throws HyracksDataException { if (canHoldNewTuple(fieldSlots.length, length)) { int effectiveSlots = 0; for (int i = 0; i < fieldSlots.length; ++i) { if (fieldSlots[i] > 0) { IntSerDeUtils.putInt(array, tupleDataEndOffset + i * 4, fieldSlots[i]); effectiveSlots++; } } System.arraycopy(bytes, offset, array, tupleDataEndOffset + effectiveSlots * 4, length); tupleDataEndOffset += effectiveSlots * 4 + length; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true; } return false; }
/** * append fieldSlots and bytes to the current frame */ @Override public boolean append(int[] fieldEndOffsets, byte[] bytes, int offset, int length) throws HyracksDataException { if (canHoldNewTuple(fieldEndOffsets.length, length)) { for (int i = 0; i < fieldEndOffsets.length; ++i) { IntSerDeUtils.putInt(array, tupleDataEndOffset + i * 4, fieldEndOffsets[i]); } System.arraycopy(bytes, offset, array, tupleDataEndOffset + fieldEndOffsets.length * 4, length); tupleDataEndOffset += fieldEndOffsets.length * 4 + length; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true; } return false; }
@Override public int flush(IFrameWriter writer) throws HyracksDataException { outputAppender.reset(outputFrame, true); int maxFrameSize = outputFrame.getFrameSize(); int limit = Math.min(tupleCount, outputLimit); int io = 0; for (int ptr = 0; ptr < limit; ++ptr) { int i = tPointers[ptr * ptrSize + ID_FRAME_ID]; int tStart = tPointers[ptr * ptrSize + ID_TUPLE_START]; int tEnd = tPointers[ptr * ptrSize + ID_TUPLE_END]; bufferManager.getFrame(i, info); inputTupleAccessor.reset(info.getBuffer(), info.getStartOffset(), info.getLength()); int flushed = FrameUtils.appendToWriter(writer, outputAppender, inputTupleAccessor, tStart, tEnd); if (flushed > 0) { maxFrameSize = Math.max(maxFrameSize, flushed); io++; } } maxFrameSize = Math.max(maxFrameSize, outputFrame.getFrameSize()); outputAppender.write(writer, true); if (LOGGER.isTraceEnabled()) { LOGGER.trace( "Flushed records:" + limit + " out of " + tupleCount + "; Flushed through " + (io + 1) + " frames"); } return maxFrameSize; }
public boolean appendField(byte[] bytes, int offset, int length) throws HyracksDataException { if (canHoldNewTuple(fieldCount, lastFieldEndOffset + length)) { int currentFieldDataStart = tupleDataEndOffset + fieldCount * 4 + lastFieldEndOffset; System.arraycopy(bytes, offset, array, currentFieldDataStart, length); lastFieldEndOffset = lastFieldEndOffset + length; IntSerDeUtils.putInt(array, tupleDataEndOffset + currentField * 4, lastFieldEndOffset); if (++currentField == fieldCount) { tupleDataEndOffset += fieldCount * 4 + lastFieldEndOffset; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); //reset for the next tuple currentField = 0; lastFieldEndOffset = 0; } return true; } else { if (currentField > 0) { copyLeftOverDataFromFrameToBuffer(); } return false; } }
@Override @SuppressWarnings("deprecation") public int flush(IFrameWriter writer) throws HyracksDataException { outputAppender.reset(outputFrame, true); int maxFrameSize = outputFrame.getFrameSize(); int numEntries = heap.getNumEntries(); IResetableComparable[] entries = heap.getEntries(); int io = 0; for (int i = 0; i < numEntries; i++) { HeapEntry minEntry = (HeapEntry) entries[i]; bufferAccessor1.reset(minEntry.tuplePointer); int flushed = FrameUtils.appendToWriter(writer, outputAppender, bufferAccessor1.getBuffer().array(), bufferAccessor1.getTupleStartOffset(), bufferAccessor1.getTupleLength()); if (flushed > 0) { maxFrameSize = Math.max(maxFrameSize, flushed); io++; } } maxFrameSize = Math.max(maxFrameSize, outputFrame.getFrameSize()); outputAppender.write(writer, true); if (LOGGER.isInfoEnabled()) { LOGGER.info("Flushed records:" + numEntries + "; Flushed through " + (io + 1) + " frames"); } return maxFrameSize; }
dataLen1); tupleDataEndOffset += (length0 + length1); IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true;
tupleDataEndOffset + slotsLen0 + slotsLen1 + dataLen0, dataLen1); tupleDataEndOffset += (length0 + length1); IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true;
tupleDataEndOffset + slotsLen0 + slotsLen1 + dataLen0, dataLen1); tupleDataEndOffset += (length0 + length1); IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true;
IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(array, FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true;
public boolean append(ITupleReference tuple) throws HyracksDataException { int length = 0; for (int i = 0; i < tuple.getFieldCount(); i++) { length += tuple.getFieldLength(i); } if (canHoldNewTuple(tuple.getFieldCount(), length)) { length = 0; for (int i = 0; i < tuple.getFieldCount(); ++i) { length += tuple.getFieldLength(i); IntSerDeUtils.putInt(array, tupleDataEndOffset + i * 4, length); } length = 0; for (int i = 0; i < tuple.getFieldCount(); ++i) { System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), array, tupleDataEndOffset + tuple.getFieldCount() * 4 + length, tuple.getFieldLength(i)); length += tuple.getFieldLength(i); } tupleDataEndOffset += tuple.getFieldCount() * 4 + length; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()) - 4 * (tupleCount + 1), tupleDataEndOffset); ++tupleCount; IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount); return true; } return false; }
while (readSize < frame.getFrameSize()) { if (currentRecordMonitor.hasMoreFrames()) { final ByteBuffer readBuffer = currentRecordChannel.getNextBuffer();
ByteBuffer copyBuffer = bufferManager.acquireFrame(rPartbuff.getFrameSize()); copyBuffer = bufferManager.acquireFrame(rPartbuff.getFrameSize());