private void startProcess() { int count = messagesInFlight.incrementAndGet(); taskMetrics.messagesInFlight().set(count); }
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
@Before public void setup() { TaskInstanceMetrics metrics = new TaskInstanceMetrics("Partition 0", new MetricsRegistryMap()); listener = new TaskCallbackListener() { @Override public void onComplete(TaskCallback callback) { } @Override public void onFailure(TaskCallback callback, Throwable t) { } }; callbackManager = new TaskCallbackManager(listener, null, -1, 2, () -> System.nanoTime()); }
taskInstanceMetrics.get(taskName) != null ? taskInstanceMetrics.get(taskName).registry() : new MetricsRegistryMap();
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
TaskInstance createTaskInstance(AsyncStreamTask task, TaskName taskName, SystemStreamPartition ssp, OffsetManager manager, SystemConsumers consumers) { TaskModel taskModel = mock(TaskModel.class); when(taskModel.getTaskName()).thenReturn(taskName); TaskInstanceMetrics taskInstanceMetrics = new TaskInstanceMetrics("task", new MetricsRegistryMap()); scala.collection.immutable.Set<SystemStreamPartition> sspSet = JavaConverters.asScalaSetConverter(Collections.singleton(ssp)).asScala().toSet(); return new TaskInstance(task, taskModel, taskInstanceMetrics, null, consumers, mock(TaskInstanceCollector.class), manager, null, null, null, sspSet, new TaskInstanceExceptionHandler(taskInstanceMetrics, new scala.collection.immutable.HashSet<String>()), null, null, null, new scala.collection.immutable.HashSet<>(), null, mock(JobContext.class), mock(ContainerContext.class), Option.apply(null), Option.apply(null), Option.apply(null)); }
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
private void startProcess() { int count = messagesInFlight.incrementAndGet(); taskMetrics.messagesInFlight().set(count); }
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
private void doneProcess() { int count = messagesInFlight.decrementAndGet(); taskMetrics.messagesInFlight().set(count); }
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
private void startProcess() { int count = messagesInFlight.incrementAndGet(); taskMetrics.messagesInFlight().set(count); }
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
/** * Insert an PendingEnvelope into the pending envelope queue. * The function will be called in the run loop thread so no synchronization. * @param pendingEnvelope */ private void insertEnvelope(PendingEnvelope pendingEnvelope) { pendingEnvelopeQueue.add(pendingEnvelope); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("Insert envelope to task {} queue.", taskName); log.debug("Task {} pending envelope count is {} after insertion.", taskName, queueSize); }
private void startProcess() { int count = messagesInFlight.incrementAndGet(); taskMetrics.messagesInFlight().set(count); }
assertEquals(5L, containerMetrics.envelopes().getCount()); assertEquals(3L, containerMetrics.processes().getCount()); assertEquals(2L, t0.metrics().asyncCallbackCompleted().getCount()); assertEquals(1L, t1.metrics().asyncCallbackCompleted().getCount());
/** * Fetch the pending envelope in the pending queue for the task to process. * Update the chooser for flow control on the SSP level. Once it's updated, the AsyncRunLoop * will be able to choose new messages from this SSP for the task to process. Note that we * update only when the envelope is first time being processed. This solves the issue in * Broadcast stream where a message need to be processed by multiple tasks. In that case, * the envelope will be in the pendingEnvelopeQueue of each task. Only the first fetch updates * the chooser with the next envelope in the broadcast stream partition. * The function will be called in the run loop thread so no synchronization. * @return */ private IncomingMessageEnvelope fetchEnvelope() { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.remove(); int queueSize = pendingEnvelopeQueue.size(); taskMetrics.pendingMessages().set(queueSize); log.trace("fetch envelope ssp {} offset {} to process.", pendingEnvelope.envelope.getSystemStreamPartition(), pendingEnvelope.envelope.getOffset()); log.debug("Task {} pending envelopes count is {} after fetching.", taskName, queueSize); if (pendingEnvelope.markProcessed()) { SystemStreamPartition partition = pendingEnvelope.envelope.getSystemStreamPartition(); consumerMultiplexer.tryUpdate(partition); log.debug("Update chooser for {}", partition); } return pendingEnvelope.envelope; } }