@Override public void run() { try { containerMetrics.commits().inc(); long startTime = clock.nanoTime(); task.commit(); containerMetrics.commitNs().update(clock.nanoTime() - startTime); state.doneCommit(); } catch (Throwable t) { log.error("Task {} commit failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} commit completed", task.taskName()); resume(); } } };
/** * Chooses an envelope from messageChooser without updating it. This enables flow control * on the SSP level, meaning the task will not get further messages for the SSP if it cannot * process it. The chooser is updated only after the callback to process is invoked, then the task * is able to process more messages. This flow control does not block. so in case of empty message chooser, * it will return null immediately without blocking, and the chooser will not poll the underlying system * consumer since there are still messages in the SystemConsumers buffer. */ private IncomingMessageEnvelope chooseEnvelope() { IncomingMessageEnvelope envelope = consumerMultiplexer.choose(false); if (envelope != null) { log.trace("Choose envelope ssp {} offset {} for processing", envelope.getSystemStreamPartition(), envelope.getOffset()); containerMetrics.envelopes().inc(); } else { log.trace("No envelope is available"); containerMetrics.nullEnvelopes().inc(); } return envelope; }
@Override public TaskCallback createCallback() { state.startProcess(); containerMetrics.processes().inc(); return callbackManager.createCallback(task.taskName(), envelope, coordinator); } };
containerMetrics.chooseNs().update(chooseNs - startNs); containerMetrics.blockNs().update(blockNs - chooseNs); containerMetrics.utilization().set(((double) activeNs) / totalNs);
@Override public void run() { try { ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); long startTime = clock.nanoTime(); task.scheduler(coordinator); containerMetrics.timerNs().update(clock.nanoTime() - startTime); coordinatorRequests.update(coordinator); state.doneScheduler(); } catch (Throwable t) { log.error("Task {} scheduler failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} scheduler completed", task.taskName()); resume(); } } };
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
containerMetrics.chooseNs().update(chooseNs - startNs); containerMetrics.blockNs().update(blockNs - chooseNs); containerMetrics.utilization().set(((double) activeNs) / totalNs);
@Test public void testEndOfStreamWithOutOfOrderProcess() throws Exception { int maxMessagesInFlight = 2; CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(2); CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1); SystemConsumers consumerMultiplexer = mock(SystemConsumers.class); when(consumerMultiplexer.pollIntervalMs()).thenReturn(10); OffsetManager offsetManager = mock(OffsetManager.class); TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch, maxMessagesInFlight); TestTask task1 = new TestTask(true, true, false, task1ProcessedMessagesLatch, maxMessagesInFlight); TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0, offsetManager, consumerMultiplexer); TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1, offsetManager, consumerMultiplexer); Map<TaskName, TaskInstance> tasks = new HashMap<>(); tasks.put(taskName0, t0); tasks.put(taskName1, t1); task0.callbackHandler = buildOutofOrderCallback(task0); AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, maxIdleMs, containerMetrics, () -> 0L, false); when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(null).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null); runLoop.run(); task0ProcessedMessagesLatch.await(); task1ProcessedMessagesLatch.await(); assertEquals(2, task0.processed); assertEquals(2, task0.completed.get()); assertEquals(1, task1.processed); assertEquals(1, task1.completed.get()); assertEquals(5L, containerMetrics.envelopes().getCount()); assertEquals(3L, containerMetrics.processes().getCount()); }
@Override public void run() { try { ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); long startTime = clock.nanoTime(); task.scheduler(coordinator); containerMetrics.timerNs().update(clock.nanoTime() - startTime); coordinatorRequests.update(coordinator); state.doneScheduler(); } catch (Throwable t) { log.error("Task {} scheduler failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} scheduler completed", task.taskName()); resume(); } } };
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
containerMetrics.chooseNs().update(chooseNs - startNs); containerMetrics.blockNs().update(blockNs - chooseNs); containerMetrics.utilization().set(((double) activeNs) / totalNs);
@Test public void testProcessOutOfOrder() throws Exception { int maxMessagesInFlight = 2; CountDownLatch task0ProcessedMessagesLatch = new CountDownLatch(2); CountDownLatch task1ProcessedMessagesLatch = new CountDownLatch(1); SystemConsumers consumerMultiplexer = mock(SystemConsumers.class); when(consumerMultiplexer.pollIntervalMs()).thenReturn(10); OffsetManager offsetManager = mock(OffsetManager.class); TestTask task0 = new TestTask(true, true, false, task0ProcessedMessagesLatch, maxMessagesInFlight); TestTask task1 = new TestTask(true, false, false, task1ProcessedMessagesLatch, maxMessagesInFlight); TaskInstance t0 = createTaskInstance(task0, taskName0, ssp0, offsetManager, consumerMultiplexer); TaskInstance t1 = createTaskInstance(task1, taskName1, ssp1, offsetManager, consumerMultiplexer); Map<TaskName, TaskInstance> tasks = new HashMap<>(); tasks.put(taskName0, t0); tasks.put(taskName1, t1); task0.callbackHandler = buildOutofOrderCallback(task0); AsyncRunLoop runLoop = new AsyncRunLoop(tasks, executor, consumerMultiplexer, maxMessagesInFlight, windowMs, commitMs, callbackTimeoutMs, maxThrottlingDelayMs, maxIdleMs, containerMetrics, () -> 0L, false); when(consumerMultiplexer.choose(false)).thenReturn(envelope0).thenReturn(envelope3).thenReturn(envelope1).thenReturn(ssp0EndOfStream).thenReturn(ssp1EndOfStream).thenReturn(null); runLoop.run(); task0ProcessedMessagesLatch.await(); task1ProcessedMessagesLatch.await(); assertEquals(2, task0.processed); assertEquals(2, task0.completed.get()); assertEquals(1, task1.processed); assertEquals(1, task1.completed.get()); assertEquals(5L, containerMetrics.envelopes().getCount()); assertEquals(3L, containerMetrics.processes().getCount()); }
/** * Chooses an envelope from messageChooser without updating it. This enables flow control * on the SSP level, meaning the task will not get further messages for the SSP if it cannot * process it. The chooser is updated only after the callback to process is invoked, then the task * is able to process more messages. This flow control does not block. so in case of empty message chooser, * it will return null immediately without blocking, and the chooser will not poll the underlying system * consumer since there are still messages in the SystemConsumers buffer. */ private IncomingMessageEnvelope chooseEnvelope() { IncomingMessageEnvelope envelope = consumerMultiplexer.choose(false); if (envelope != null) { log.trace("Choose envelope ssp {} offset {} for processing", envelope.getSystemStreamPartition(), envelope.getOffset()); containerMetrics.envelopes().inc(); } else { log.trace("No envelope is available"); containerMetrics.nullEnvelopes().inc(); } return envelope; }
@Override public void run() { try { containerMetrics.commits().inc(); long startTime = clock.nanoTime(); task.commit(); containerMetrics.commitNs().update(clock.nanoTime() - startTime); state.doneCommit(); } catch (Throwable t) { log.error("Task {} commit failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} commit completed", task.taskName()); resume(); } } };
@Override public TaskCallback createCallback() { state.startProcess(); containerMetrics.processes().inc(); return callbackManager.createCallback(task.taskName(), envelope, coordinator); } };
@Override public void run() { try { ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); long startTime = clock.nanoTime(); task.scheduler(coordinator); containerMetrics.timerNs().update(clock.nanoTime() - startTime); coordinatorRequests.update(coordinator); state.doneScheduler(); } catch (Throwable t) { log.error("Task {} scheduler failed", task.taskName(), t); abort(t); } finally { log.trace("Task {} scheduler completed", task.taskName()); resume(); } } };
@Override public void run() { try { state.doneProcess(); state.taskMetrics.asyncCallbackCompleted().inc(); TaskCallbackImpl callbackImpl = (TaskCallbackImpl) callback; containerMetrics.processNs().update(clock.nanoTime() - callbackImpl.timeCreatedNs); log.trace("Got callback complete for task {}, ssp {}", callbackImpl.taskName, callbackImpl.envelope.getSystemStreamPartition()); List<TaskCallbackImpl> callbacksToUpdate = callbackManager.updateCallback(callbackImpl); for (TaskCallbackImpl callbackToUpdate : callbacksToUpdate) { IncomingMessageEnvelope envelope = callbackToUpdate.envelope; log.trace("Update offset for ssp {}, offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); // update offset task.offsetManager().update(task.taskName(), envelope.getSystemStreamPartition(), envelope.getOffset()); // update coordinator coordinatorRequests.update(callbackToUpdate.coordinator); } } catch (Throwable t) { log.error("Error marking process as complete.", t); abort(t); } finally { resume(); } } }, workNanos);
containerMetrics.chooseNs().update(chooseNs - startNs); containerMetrics.blockNs().update(blockNs - chooseNs); containerMetrics.utilization().set(((double) activeNs) / totalNs);