private void moveMessagesToTheirQueue(SystemStreamPartition ssp, List<IncomingMessageEnvelope> envelopes) { long nextOffset = nextOffsets.get(ssp); for (IncomingMessageEnvelope env : envelopes) { sink.addMessage(ssp, env); // move message to the BlockingEnvelopeMap's queue LOG.trace("IncomingMessageEnvelope. got envelope with offset:{} for ssp={}", env.getOffset(), ssp); nextOffset = Long.valueOf(env.getOffset()) + 1; } nextOffsets.put(ssp, nextOffset); }
private void moveMessagesToTheirQueue(SystemStreamPartition ssp, List<IncomingMessageEnvelope> envelopes) { long nextOffset = nextOffsets.get(ssp); for (IncomingMessageEnvelope env : envelopes) { sink.addMessage(ssp, env); // move message to the BlockingEnvelopeMap's queue LOG.trace("IncomingMessageEnvelope. got envelope with offset:{} for ssp={}", env.getOffset(), ssp); nextOffset = Long.valueOf(env.getOffset()) + 1; } nextOffsets.put(ssp, nextOffset); }
private void moveMessagesToTheirQueue(SystemStreamPartition ssp, List<IncomingMessageEnvelope> envelopes) { long nextOffset = nextOffsets.get(ssp); for (IncomingMessageEnvelope env : envelopes) { sink.addMessage(ssp, env); // move message to the BlockingEnvelopeMap's queue LOG.trace("IncomingMessageEnvelope. got envelope with offset:{} for ssp={}", env.getOffset(), ssp); nextOffset = Long.valueOf(env.getOffset()) + 1; } nextOffsets.put(ssp, nextOffset); }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void complete() { if (scheduledFuture != null) { scheduledFuture.cancel(true); } log.trace("Callback complete for task {}, ssp {}, offset {}.", new Object[] {taskName, envelope.getSystemStreamPartition(), envelope.getOffset()}); if (isComplete.compareAndSet(false, true)) { listener.onComplete(this); } else { String msg = String.format("Callback complete was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg)); } }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
/** * Processes the incoming side input message envelope and updates the last processed offset for its SSP. * * @param message incoming message to be processed */ public void process(IncomingMessageEnvelope message) { SystemStreamPartition ssp = message.getSystemStreamPartition(); Set<String> storeNames = sspsToStores.get(ssp); for (String storeName : storeNames) { SideInputsProcessor sideInputsProcessor = storeToProcessor.get(storeName); KeyValueStore keyValueStore = (KeyValueStore) stores.get(storeName); Collection<Entry<?, ?>> entriesToBeWritten = sideInputsProcessor.process(message, keyValueStore); keyValueStore.putAll(ImmutableList.copyOf(entriesToBeWritten)); } // update the last processed offset lastProcessedOffsets.put(ssp, message.getOffset()); }
@Override public void failure(Throwable t) { if (scheduledFuture != null) { scheduledFuture.cancel(true); } if (isComplete.compareAndSet(false, true)) { String msg = String.format("Callback failed for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new SamzaException(msg, t)); } else { String msg = String.format("Task callback failure was invoked after completion for task %s, ssp %s, offset %s.", taskName, envelope.getSystemStreamPartition(), envelope.getOffset()); listener.onFailure(this, new IllegalStateException(msg, t)); } }
/** * Process asynchronously. The callback needs to be fired once the processing is done. */ private void process() { final IncomingMessageEnvelope envelope = state.fetchEnvelope(); log.trace("Process ssp {} offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); final ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); TaskCallbackFactory callbackFactory = new TaskCallbackFactory() { @Override public TaskCallback createCallback() { state.startProcess(); containerMetrics.processes().inc(); return callbackManager.createCallback(task.taskName(), envelope, coordinator); } }; task.process(envelope, coordinator, callbackFactory); }
/** * Process asynchronously. The callback needs to be fired once the processing is done. */ private void process() { final IncomingMessageEnvelope envelope = state.fetchEnvelope(); log.trace("Process ssp {} offset {}", envelope.getSystemStreamPartition(), envelope.getOffset()); final ReadableCoordinator coordinator = new ReadableCoordinator(task.taskName()); TaskCallbackFactory callbackFactory = new TaskCallbackFactory() { @Override public TaskCallback createCallback() { state.startProcess(); containerMetrics.processes().inc(); return callbackManager.createCallback(task.taskName(), envelope, coordinator); } }; task.process(envelope, coordinator, callbackFactory); }
private void verifyEvents(List<IncomingMessageEnvelope> messages, List<EventData> eventDataList, Interceptor interceptor) { Assert.assertEquals(messages.size(), eventDataList.size()); for (int i = 0; i < messages.size(); i++) { IncomingMessageEnvelope message = messages.get(i); EventData eventData = eventDataList.get(i); Assert.assertEquals(message.getKey(), eventData.getSystemProperties().getPartitionKey()); Assert.assertEquals(message.getMessage(), interceptor.intercept(eventData.getBytes())); Assert.assertEquals(message.getOffset(), eventData.getSystemProperties().getOffset()); } }
private void verifyRecords(List<IncomingMessageEnvelope> outputRecords, List<Record> inputRecords, String shardId) { Iterator outputRecordsIter = outputRecords.iterator(); inputRecords.forEach(record -> { IncomingMessageEnvelope envelope = (IncomingMessageEnvelope) outputRecordsIter.next(); String outputKey = (String) envelope.getKey(); KinesisIncomingMessageEnvelope kinesisMessageEnvelope = (KinesisIncomingMessageEnvelope) envelope; Assert.assertEquals(outputKey, record.getPartitionKey()); Assert.assertEquals(kinesisMessageEnvelope.getSequenceNumber(), record.getSequenceNumber()); Assert.assertEquals(kinesisMessageEnvelope.getApproximateArrivalTimestamp(), record.getApproximateArrivalTimestamp()); Assert.assertEquals(kinesisMessageEnvelope.getShardId(), shardId); ByteBuffer outputData = ByteBuffer.wrap((byte[]) kinesisMessageEnvelope.getMessage()); record.getData().rewind(); Assert.assertTrue(outputData.equals(record.getData())); verifyOffset(envelope.getOffset(), record, shardId); }); }