private void subtractSizeOnQDrain(SystemStreamPartition systemStreamPartition, List<IncomingMessageEnvelope> outgoingList) { long outgoingListBytes = 0; for (IncomingMessageEnvelope envelope : outgoingList) { outgoingListBytes += envelope.getSize(); } // subtract the size of the messages dequeued. bufferedMessagesSize.get(systemStreamPartition).addAndGet(-1 * outgoingListBytes); }
private void subtractSizeOnQDrain(SystemStreamPartition systemStreamPartition, List<IncomingMessageEnvelope> outgoingList) { long outgoingListBytes = 0; for (IncomingMessageEnvelope envelope : outgoingList) { outgoingListBytes += envelope.getSize(); } // subtract the size of the messages dequeued. bufferedMessagesSize.get(systemStreamPartition).addAndGet(-1 * outgoingListBytes); }
/** * Place a new {@link org.apache.samza.system.IncomingMessageEnvelope} on the * queue for the specified {@link org.apache.samza.system.SystemStreamPartition}. * * @param systemStreamPartition SystemStreamPartition that owns the envelope * @param envelope Message for specified SystemStreamPartition * @throws InterruptedException from underlying concurrent collection */ protected void put(SystemStreamPartition systemStreamPartition, IncomingMessageEnvelope envelope) throws InterruptedException { bufferedMessages.get(systemStreamPartition).put(envelope); bufferedMessagesSize.get(systemStreamPartition).addAndGet(envelope.getSize()); }
/** * Place a new {@link org.apache.samza.system.IncomingMessageEnvelope} on the * queue for the specified {@link org.apache.samza.system.SystemStreamPartition}. * * @param systemStreamPartition SystemStreamPartition that owns the envelope * @param envelope Message for specified SystemStreamPartition * @throws InterruptedException from underlying concurrent collection */ protected void put(SystemStreamPartition systemStreamPartition, IncomingMessageEnvelope envelope) throws InterruptedException { bufferedMessages.get(systemStreamPartition).put(envelope); bufferedMessagesSize.get(systemStreamPartition).addAndGet(envelope.getSize()); }
public IncomingMessageEnvelope readNext() { if (!hasNext()) { LOG.warn("Attempting to read more data when there aren't any. ssp=" + systemStreamPartition); return null; } // record the next offset before we read, so when the read fails and we reconnect, // we seek to the same offset that we try below curSingleFileOffset = curReader.nextOffset(); IncomingMessageEnvelope messageEnvelope = curReader.readNext(); // Copy everything except for the offset. Turn the single-file style offset into a multi-file one return new IncomingMessageEnvelope(messageEnvelope.getSystemStreamPartition(), getCurOffset(), messageEnvelope.getKey(), messageEnvelope.getMessage(), messageEnvelope.getSize(), messageEnvelope.getEventTime(), messageEnvelope.getArrivalTime()); }