/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
/** * Fetch the pending envelope in the pending queue for the task to process. * Update the chooser for flow control on the SSP level. Once it's updated, the AsyncRunLoop * will be able to choose new messages from this SSP for the task to process. Note that we * update only when the envelope is first time being processed. This solves the issue in * Broadcast stream where a message need to be processed by multiple tasks. In that case, * the envelope will be in the pendingEnvelopeQueue of each task. Only the first fetch updates * the chooser with the next envelope in the broadcast stream partition. * The function will be called in the run loop thread so no synchronization. * @return */ private IncomingMessageEnvelope fetchEnvelope() { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.remove(); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("fetch envelope ssp {} offset {} to process.", pendingEnvelope.envelope.getSystemStreamPartition(), pendingEnvelope.envelope.getOffset()); log.debug("Task {} pending envelopes count is {} after fetching.", taskName, queueSize); if (pendingEnvelope.markProcessed()) { SystemStreamPartition partition = pendingEnvelope.envelope.getSystemStreamPartition(); consumerMultiplexer.tryUpdate(partition); log.debug("Update chooser for {}", partition); } return pendingEnvelope.envelope; } }
/** * Fetch the pending envelope in the pending queue for the task to process. * Update the chooser for flow control on the SSP level. Once it's updated, the AsyncRunLoop * will be able to choose new messages from this SSP for the task to process. Note that we * update only when the envelope is first time being processed. This solves the issue in * Broadcast stream where a message need to be processed by multiple tasks. In that case, * the envelope will be in the pendingEnvelopeQueue of each task. Only the first fetch updates * the chooser with the next envelope in the broadcast stream partition. * The function will be called in the run loop thread so no synchronization. * @return */ private IncomingMessageEnvelope fetchEnvelope() { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.remove(); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("fetch envelope ssp {} offset {} to process.", pendingEnvelope.envelope.getSystemStreamPartition(), pendingEnvelope.envelope.getOffset()); log.debug("Task {} pending envelopes count is {} after fetching.", taskName, queueSize); if (pendingEnvelope.markProcessed()) { SystemStreamPartition partition = pendingEnvelope.envelope.getSystemStreamPartition(); consumerMultiplexer.tryUpdate(partition); log.debug("Update chooser for {}", partition); } return pendingEnvelope.envelope; } }
/** * Fetch the pending envelope in the pending queue for the task to process. * Update the chooser for flow control on the SSP level. Once it's updated, the AsyncRunLoop * will be able to choose new messages from this SSP for the task to process. Note that we * update only when the envelope is first time being processed. This solves the issue in * Broadcast stream where a message need to be processed by multiple tasks. In that case, * the envelope will be in the pendingEnvelopeQueue of each task. Only the first fetch updates * the chooser with the next envelope in the broadcast stream partition. * The function will be called in the run loop thread so no synchronization. * @return */ private IncomingMessageEnvelope fetchEnvelope() { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.remove(); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("fetch envelope ssp {} offset {} to process.", pendingEnvelope.envelope.getSystemStreamPartition(), pendingEnvelope.envelope.getOffset()); log.debug("Task {} pending envelopes count is {} after fetching.", taskName, queueSize); if (pendingEnvelope.markProcessed()) { SystemStreamPartition partition = pendingEnvelope.envelope.getSystemStreamPartition(); consumerMultiplexer.tryUpdate(partition); log.debug("Update chooser for {}", partition); } return pendingEnvelope.envelope; } }
/** * Fetch the pending envelope in the pending queue for the task to process. * Update the chooser for flow control on the SSP level. Once it's updated, the AsyncRunLoop * will be able to choose new messages from this SSP for the task to process. Note that we * update only when the envelope is first time being processed. This solves the issue in * Broadcast stream where a message need to be processed by multiple tasks. In that case, * the envelope will be in the pendingEnvelopeQueue of each task. Only the first fetch updates * the chooser with the next envelope in the broadcast stream partition. * The function will be called in the run loop thread so no synchronization. * @return */ private IncomingMessageEnvelope fetchEnvelope() { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.remove(); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("fetch envelope ssp {} offset {} to process.", pendingEnvelope.envelope.getSystemStreamPartition(), pendingEnvelope.envelope.getOffset()); log.debug("Task {} pending envelopes count is {} after fetching.", taskName, queueSize); if (pendingEnvelope.markProcessed()) { SystemStreamPartition partition = pendingEnvelope.envelope.getSystemStreamPartition(); consumerMultiplexer.tryUpdate(partition); log.debug("Update chooser for {}", partition); } return pendingEnvelope.envelope; } }
/** * Fetch the pending envelope in the pending queue for the task to process. * Update the chooser for flow control on the SSP level. Once it's updated, the AsyncRunLoop * will be able to choose new messages from this SSP for the task to process. Note that we * update only when the envelope is first time being processed. This solves the issue in * Broadcast stream where a message need to be processed by multiple tasks. In that case, * the envelope will be in the pendingEnvelopeQueue of each task. Only the first fetch updates * the chooser with the next envelope in the broadcast stream partition. * The function will be called in the run loop thread so no synchronization. * @return */ private IncomingMessageEnvelope fetchEnvelope() { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.remove(); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("fetch envelope ssp {} offset {} to process.", pendingEnvelope.envelope.getSystemStreamPartition(), pendingEnvelope.envelope.getOffset()); log.debug("Task {} pending envelopes count is {} after fetching.", taskName, queueSize); if (pendingEnvelope.markProcessed()) { SystemStreamPartition partition = pendingEnvelope.envelope.getSystemStreamPartition(); consumerMultiplexer.tryUpdate(partition); log.debug("Update chooser for {}", partition); } return pendingEnvelope.envelope; } }