/** * Insert the envelope into the task pending queues and run all the tasks */ private void runTasks(IncomingMessageEnvelope envelope) { if (envelope != null) { PendingEnvelope pendingEnvelope = new PendingEnvelope(envelope); for (AsyncTaskWorker worker : sspToTaskWorkerMapping.get(envelope.getSystemStreamPartition())) { worker.state.insertEnvelope(pendingEnvelope); } } for (AsyncTaskWorker worker: taskWorkers) { worker.run(); } }
/** * Insert the envelope into the task pending queues and run all the tasks */ private void runTasks(IncomingMessageEnvelope envelope) { if (envelope != null) { PendingEnvelope pendingEnvelope = new PendingEnvelope(envelope); for (AsyncTaskWorker worker : sspToTaskWorkerMapping.get(envelope.getSystemStreamPartition())) { worker.state.insertEnvelope(pendingEnvelope); } } for (AsyncTaskWorker worker: taskWorkers) { worker.run(); } }
/** * Insert the envelope into the task pending queues and run all the tasks */ private void runTasks(IncomingMessageEnvelope envelope) { if (envelope != null) { PendingEnvelope pendingEnvelope = new PendingEnvelope(envelope); for (AsyncTaskWorker worker : sspToTaskWorkerMapping.get(envelope.getSystemStreamPartition())) { worker.state.insertEnvelope(pendingEnvelope); } } for (AsyncTaskWorker worker: taskWorkers) { worker.run(); } }
/** * Insert the envelope into the task pending queues and run all the tasks */ private void runTasks(IncomingMessageEnvelope envelope) { if (envelope != null) { PendingEnvelope pendingEnvelope = new PendingEnvelope(envelope); for (AsyncTaskWorker worker : sspToTaskWorkerMapping.get(envelope.getSystemStreamPartition())) { worker.state.insertEnvelope(pendingEnvelope); } } for (AsyncTaskWorker worker: taskWorkers) { worker.run(); } }
/** * Insert the envelope into the task pending queues and run all the tasks */ private void runTasks(IncomingMessageEnvelope envelope) { if (envelope != null) { PendingEnvelope pendingEnvelope = new PendingEnvelope(envelope); for (AsyncTaskWorker worker : sspToTaskWorkerMapping.get(envelope.getSystemStreamPartition())) { worker.state.insertEnvelope(pendingEnvelope); } } for (AsyncTaskWorker worker: taskWorkers) { worker.run(); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
private boolean checkEndOfStream() { if (pendingEnvelopeQueue.size() == 1) { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.peek(); IncomingMessageEnvelope envelope = pendingEnvelope.envelope; if (envelope.isEndOfStream()) { SystemStreamPartition ssp = envelope.getSystemStreamPartition(); processingSspSet.remove(ssp); if (!hasIntermediateStreams) { pendingEnvelopeQueue.remove(); } } } return processingSspSet.isEmpty(); }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
private boolean checkEndOfStream() { if (pendingEnvelopeQueue.size() == 1) { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.peek(); IncomingMessageEnvelope envelope = pendingEnvelope.envelope; if (envelope.isEndOfStream()) { SystemStreamPartition ssp = envelope.getSystemStreamPartition(); processingSspSet.remove(ssp); if (!hasIntermediateStreams) { pendingEnvelopeQueue.remove(); } } } return processingSspSet.isEmpty(); }
private boolean checkEndOfStream() { if (pendingEnvelopeQueue.size() == 1) { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.peek(); IncomingMessageEnvelope envelope = pendingEnvelope.envelope; if (envelope.isEndOfStream()) { SystemStreamPartition ssp = envelope.getSystemStreamPartition(); processingSspSet.remove(ssp); if (!hasIntermediateStreams) { pendingEnvelopeQueue.remove(); } } } return processingSspSet.isEmpty(); }
private boolean checkEndOfStream() { if (pendingEnvelopeQueue.size() == 1) { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.peek(); IncomingMessageEnvelope envelope = pendingEnvelope.envelope; if (envelope.isEndOfStream()) { SystemStreamPartition ssp = envelope.getSystemStreamPartition(); processingSspSet.remove(ssp); if (!hasIntermediateStreams) { pendingEnvelopeQueue.remove(); } } } return processingSspSet.isEmpty(); }
private boolean checkEndOfStream() { if (pendingEnvelopeQueue.size() == 1) { PendingEnvelope pendingEnvelope = pendingEnvelopeQueue.peek(); IncomingMessageEnvelope envelope = pendingEnvelope.envelope; if (envelope.isEndOfStream()) { SystemStreamPartition ssp = envelope.getSystemStreamPartition(); processingSspSet.remove(ssp); if (!hasIntermediateStreams) { pendingEnvelopeQueue.remove(); } } } return processingSspSet.isEmpty(); }
/** * Process asynchronously. The callback needs to be fired once the processing is done. */ private void process() { final IncomingMessageEnvelope envelope = state.fetchEnvelope(); log.trace("Process ssp {} offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); final ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); TaskCallbackFactory callbackFactory = new TaskCallbackFactory() { @Override public TaskCallback createCallback() { state.startProcess(); containerMetrics.processes().inc(); return callbackManager.createCallback(task.taskName(), envelope, coordinator); } }; task.process(envelope, coordinator, callbackFactory); }
public IncomingMessageEnvelope readNext() { if (!hasNext()) { LOG.warn("Attempting to read more data when there aren't any. ssp=" + systemStreamPartition); return null; } // record the next offset before we read, so when the read fails and we reconnect, // we seek to the same offset that we try below curSingleFileOffset = curReader.nextOffset(); IncomingMessageEnvelope messageEnvelope = curReader.readNext(); // Copy everything except for the offset. Turn the single-file style offset into a multi-file one return new IncomingMessageEnvelope(messageEnvelope.getSystemStreamPartition(), getCurOffset(), messageEnvelope.getKey(), messageEnvelope.getMessage(), messageEnvelope.getSize(), messageEnvelope.getEventTime(), messageEnvelope.getArrivalTime()); }